From 3c08d0446d9e0e5ac11286874fffa4311659e0e4 Mon Sep 17 00:00:00 2001 From: Beliy Nikita Date: Fri, 19 Feb 2021 13:08:13 +0100 Subject: [PATCH 01/36] Added function to generate derivatives --- +bids/derivate.m | 138 +++++++++++++++++++++++++++++++++++++++++++++++ .gitignore | 3 +- 2 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 +bids/derivate.m diff --git a/+bids/derivate.m b/+bids/derivate.m new file mode 100644 index 00000000..524389ec --- /dev/null +++ b/+bids/derivate.m @@ -0,0 +1,138 @@ +function derivatives = derivate(BIDS, out_path, name, varargin) + % + % Copy selected data from BIDS layout to given derivatives folder, + % returning layout of new derivatives folder + % + % USAGE:: + % + % derivatives = derivate(BIDS, out_path, ...) + % + % :param BIDS: BIDS directory name or BIDS structure (from bids.layout) + % :type BIDS: (strcuture or string) + % :param out_path: path to directory containing the derivatives + % :type out_path: string + % :param name: name of pipeline to use + % :type name: string + % + % + % __________________________________________________________________________ + % + % BIDS (Brain Imaging Data Structure): https://bids.neuroimaging.io/ + % The brain imaging data structure, a format for organizing and + % describing outputs of neuroimaging experiments. + % K. J. Gorgolewski et al, Scientific Data, 2016. + % __________________________________________________________________________ + + % Copyright (C) 2016-2018, Guillaume Flandin, Wellcome Centre for Human Neuroimaging + % Copyright (C) 2018--, BIDS-MATLAB developers + + narginchk(3, Inf); + + BIDS = bids.layout(BIDS); + + if ~exist(out_path, 'dir') + error(['Output path ' out_path ' not found']); + end + + derivatives = []; + data_list = bids.query(BIDS, 'data', varargin{:}); + subjects_list = bids.query(BIDS, 'subjects', varargin{:}); + + if isempty(data_list) + warning(['No data found for this query']); + return; + else + fprintf('Found %d files in %d subjects\n', length(data_list), length(subjects_list)); + end + + pth_BIDSderiv = fullfile(out_path, name); + if ~exist(pth_BIDSderiv,'dir') + mkdir(pth_BIDSderiv); + end + + % creating description + descr_file = fullfile(pth_BIDSderiv, 'dataset_description.json'); + pipeline.Name = mfilename; + % pipeline.Verion = ? + pipeline.Container = varargin; + + % loading dataset description + if exist(descr_file, 'file') + description = bids.util.jsondecode(descr_file); + else + description = BIDS.description; + end + + % updating GeneratedBy + if isfield(description, 'GeneratedBy') + description.GeneratedBy = [description.GeneratedBy pipeline]; + else + description.GeneratedBy = [pipeline]; + end + + bids.util.jsonencode(descr_file, description, 'Indent', ' '); + + % extracting participants.tsv file? + + % looping over selected files + for iFile = 1:numel(data_list) + copy_file(BIDS, pth_BIDSderiv, data_list{iFile}); + end + +end + +function status = copy_file(BIDS, derivatives_folder, data_file) + status = 1; + info = bids.internal.return_file_info(BIDS, data_file); + file = BIDS.subjects(info.sub_idx).(info.modality)(info.file_idx); + basename = file.filename(1:end - length(file.ext)); + out_dir = fullfile(derivatives_folder,... + BIDS.subjects(info.sub_idx).name,... + BIDS.subjects(info.sub_idx).session,... + info.modality); + out_path = fullfile(out_dir, basename); + meta_file = [out_path '.json']; + + % ignore already existing files; avoid circular references + if exist(meta_file) + return + end + if ~exist(out_dir, 'dir') + mkdir(out_dir); + end + % copy data file + if endsWith(file.ext, '.gz') + gunzip(data_file, out_dir); + else + [status,message,messageId] = copyfile(data_file, [out_path file.ext]); + end + if ~status + warning([messageId ': ' message]); + return; + end + % export metadata + if ~strcmpi(file.ext, '.json') % skip if data file is json + bids.util.jsonencode(meta_file, file.meta); + end + + % checking that json is created + if ~exist(meta_file) + error(['Failed to create sidecar json file: ' meta_file]); + end + + % trating depandencies + if ~isempty(file.dependencies) + dependencies = fieldnames(file.dependencies); + for dep = 1:numel(dependencies) + for idep = 1: numel(file.dependencies.(dependencies{dep})) + dep_file = file.dependencies.(dependencies{dep}){idep}; + if exist(dep_file, 'file') + copy_file(BIDS, derivatives_folder, dep_file); + else + warning(['Dependency file ' dep_file ' not found']); + end + end + end + end + +end diff --git a/.gitignore b/.gitignore index 6a705523..efb87036 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.asv *.*~ +*.swp /tests/bids-examples /tests/*.tsv @@ -10,4 +11,4 @@ env/* # ignore bids-specification repo that gets cloned during CI bids-specification/* -.prettierrc \ No newline at end of file +.prettierrc From a9ce261ff35cc45803637287dc82dae2647245a9 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 27 Feb 2021 21:05:52 +0100 Subject: [PATCH 02/36] refactor get_metadata --- +bids/+internal/get_meta_list.m | 82 +++++++++++++++++++++++++++++++ +bids/+internal/get_metadata.m | 86 ++++----------------------------- 2 files changed, 92 insertions(+), 76 deletions(-) create mode 100644 +bids/+internal/get_meta_list.m diff --git a/+bids/+internal/get_meta_list.m b/+bids/+internal/get_meta_list.m new file mode 100644 index 00000000..5b28c617 --- /dev/null +++ b/+bids/+internal/get_meta_list.m @@ -0,0 +1,82 @@ +function metalist = get_meta_list(filename, pattern) + % + % Read a BIDS's file metadata according to the inheritance principle + % + % USAGE:: + % + % meta = bids.internal.get_metadata(filename, pattern = '^.*%s\\.json$') + % + % :param filename: fullpath name of file following BIDS standard + % :type filename: string + % :param pattern: Regular expression matching the metadata file (default is ``'^.*%s\\.json$'``) + % If provided, it must at least be ``'%s'``. + % :type pattern: string + % + % + % metalist - list of paths to metafiles + % __________________________________________________________________________ + + % Copyright (C) 2016-2018, Guillaume Flandin, Wellcome Centre for Human Neuroimaging + % Copyright (C) 2018--, BIDS-MATLAB developers + + if nargin == 1 + pattern = '^.*%s\\.json$'; + end + + pth = fileparts(filename); + p = bids.internal.parse_filename(filename); + metalist = {}; + + N = 3; + + % -There is a session level in the hierarchy + if isfield(p.entities, 'ses') && ~isempty(p.entities.ses) + N = N + 1; + end + + % -Loop from the directory where the file of interest is back to the + % top level of the BIDS hierarchy + for n = 1:N + + % -List the potential metadata files associated with this file suffix type + % Default is to assume it is a JSON file + metafile = bids.internal.file_utils('FPList', pth, sprintf(pattern, p.suffix)); + + if isempty(metafile) + metafile = {}; + else + metafile = cellstr(metafile); + end + + % -For all those files we find which one is potentially associated with + % the file of interest + for i = 1:numel(metafile) + + p2 = bids.internal.parse_filename(metafile{i}); + entities = {}; + if isfield(p2, 'entities') + entities = fieldnames(p2.entities); + end + + % -Check if this metadata file contains the same entity-label pairs as its + % data file counterpart + ismeta = true; + for j = 1:numel(entities) + if ~isfield(p.entities, entities{j}) || ... + ~strcmp(p.entities.(entities{j}), p2.entities.(entities{j})) + ismeta = false; + break + end + end + + % append path to list + if ismeta + metalist{end + 1} = metafile{i}; + end + + end + + % -Go up to the parent folder + pth = fullfile(pth, '..'); + end +end diff --git a/+bids/+internal/get_metadata.m b/+bids/+internal/get_metadata.m index 1c0759f1..cc8026b4 100644 --- a/+bids/+internal/get_metadata.m +++ b/+bids/+internal/get_metadata.m @@ -1,16 +1,13 @@ -function meta = get_metadata(filename, pattern) +function meta = get_metadata(metafile) % % Read a BIDS's file metadata according to the inheritance principle % % USAGE:: % - % meta = bids.internal.get_metadata(filename, pattern = '^.*%s\\.json$') + % meta = bids.internal.get_metadata(metafile) % - % :param filename: fullpath name of file following BIDS standard - % :type filename: string - % :param pattern: Regular expression matching the metadata file (default is ``'^.*%s\\.json$'``) - % If provided, it must at least be ``'%s'``. - % :type pattern: string + % :param metafile: list of fullpath names of metafiles + % :type metafile: string or array of strings % % % meta - metadata structure @@ -19,83 +16,20 @@ % Copyright (C) 2016-2018, Guillaume Flandin, Wellcome Centre for Human Neuroimaging % Copyright (C) 2018--, BIDS-MATLAB developers - % assume most files are of the form *_suffix.json - % add an exception for files with no suffix, like participants.tsv - if nargin == 1 - pattern = '^.*_?%s\\.json$'; - end - - pth = fileparts(filename); - p = bids.internal.parse_filename(filename); - meta = struct(); + metafile = cellstr(metafile); - N = 3; - - % -There is a session level in the hierarchy - if isfield(p, 'entities') && isfield(p.entities, 'ses') && ~isempty(p.entities.ses) - N = N + 1; - end - - % -Loop from the directory where the file of interest is back to the - % top level of the BIDS hierarchy - for n = 1:N - - % -List the potential metadata files associated with this file suffix type - % Default is to assume it is a JSON file - metafile = bids.internal.file_utils('FPList', pth, sprintf(pattern, p.suffix)); - - if isempty(metafile) - metafile = {}; + for i = 1:numel(metafile) + if bids.internal.endsWith(metafile{i}, '.json') + meta = update_metadata(meta, bids.util.jsondecode(metafile{i}), metafile{i}); else - metafile = cellstr(metafile); + meta.filename = metafile{i}; end - % -For all those files we find which one is potentially associated with - % the file of interest - for i = 1:numel(metafile) - - p2 = bids.internal.parse_filename(metafile{i}); - entities = {}; - if isfield(p2, 'entities') - entities = fieldnames(p2.entities); - end - - % Check if this metadata file contains - % - the same entity-label pairs - % - same prefix - % as its data file counterpart - ismeta = true; - if ~strcmp(p.suffix, p2.suffix) - ismeta = false; - end - for j = 1:numel(entities) - if ~isfield(p.entities, entities{j}) || ... - ~strcmp(p.entities.(entities{j}), p2.entities.(entities{j})) - ismeta = false; - break - end - end - - % -Read the content of the metadata file if it is a JSON file and update - % the metadata concerning the file of interest otherwise store the filename - if ismeta - if strcmp(p2.ext, '.json') - meta = update_metadata(meta, bids.util.jsondecode(metafile{i}), metafile{i}); - else - meta.filename = metafile{i}; - end - end - - end - - % -Go up to the parent folder - pth = fullfile(pth, '..'); - end if isempty(meta) - warning('No metadata for %s', filename); + warning('No metadata for %s', metafile); end end From f3636725dfca76d402cad9e9df46b6dd7cc59aaf Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 27 Feb 2021 21:06:11 +0100 Subject: [PATCH 03/36] reorganize dependencies --- +bids/+internal/endsWith.m | 23 +++++ +bids/+internal/startsWith.m | 23 +++++ +bids/layout.m | 182 +++++++++++++++-------------------- 3 files changed, 126 insertions(+), 102 deletions(-) create mode 100644 +bids/+internal/endsWith.m create mode 100644 +bids/+internal/startsWith.m diff --git a/+bids/+internal/endsWith.m b/+bids/+internal/endsWith.m new file mode 100644 index 00000000..a6cf7fc0 --- /dev/null +++ b/+bids/+internal/endsWith.m @@ -0,0 +1,23 @@ +function res = endsWith(str, pat) + % + % Checks id character array 'str' ends with 'pat' + % + % USAGE res = bids.internal.endsWith(str, pat) + % + % str - character array + % pat - character array + % + % __________________________________________________________________________ + % + % Based on spm_file.m and spm_select.m from SPM12. + % __________________________________________________________________________ + + % Copyright (C) 2011-2018 Guillaume Flandin, Wellcome Centre for Human Neuroimaging + res = false; + l_pat = length(pat); + if l_pat > length(str) + return; + end + res = strcmp(str(end - l_pat +1 : end), pat); + +end diff --git a/+bids/+internal/startsWith.m b/+bids/+internal/startsWith.m new file mode 100644 index 00000000..7cf84d17 --- /dev/null +++ b/+bids/+internal/startsWith.m @@ -0,0 +1,23 @@ +function res = startsWith(str, pat) + % + % Checks id character array 'str' starts with 'pat' + % + % USAGE res = bids.internal.startsWith(str, pat) + % + % str - character array + % pat - character array + % + % __________________________________________________________________________ + % + % Based on spm_file.m and spm_select.m from SPM12. + % __________________________________________________________________________ + + % Copyright (C) 2011-2018 Guillaume Flandin, Wellcome Centre for Human Neuroimaging + res = false; + l_pat = length(pat); + if l_pat > length(str) + return; + end + res = strcmp(str(1 : l_pat), pat); + +end diff --git a/+bids/layout.m b/+bids/layout.m index 920ba205..dc9f447c 100644 --- a/+bids/layout.m +++ b/+bids/layout.m @@ -125,7 +125,8 @@ %% Dependencies % ========================================================================== - BIDS = manage_intended_for(BIDS); + % BIDS = manage_intended_for(BIDS); + BIDS = manage_dependencies(BIDS); end @@ -165,12 +166,8 @@ % so the parsing is unconstrained for iModality = 1:numel(modalities) switch modalities{iModality} - case {'anat', 'func', 'beh', 'meg', 'eeg', 'ieeg', 'pet'} + case {'anat', 'func', 'beh', 'meg', 'eeg', 'ieeg', 'pet', 'fmap', 'dwi'} subject = parse_using_schema(subject, modalities{iModality}, schema); - case 'dwi' - subject = parse_dwi(subject, schema); - case 'fmap' - subject = parse_fmap(subject, schema); case 'perf' subject = parse_perf(subject, schema); otherwise @@ -197,57 +194,36 @@ for i = 1:numel(file_list) + fullpath_filename = fullfile(pth, file_list{i}); subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema); - - end - - end - -end - -function subject = parse_dwi(subject, schema) - - % -------------------------------------------------------------------------- - % Diffusion imaging data - % -------------------------------------------------------------------------- - - modality = 'dwi'; - pth = fullfile(subject.path, modality); - - if exist(pth, 'dir') - - subject = bids.internal.add_missing_field(subject, modality); - - file_list = return_file_list(modality, subject, schema); - - for i = 1:numel(file_list) - - subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema); - - % if this file is a nifti image we add the bval and bvec as dependencies - if ~isempty(subject.(modality)) && ... - any(strcmp(subject.(modality)(end).ext, {'.nii', '.nii.gz'})) - - fullpath_filename = fullfile(subject.path, modality, file_list{i}); - - % bval & bvec file - % ------------------------------------------------------------------ - % TODO: they can also be stored at higher levels (inheritance principle) - % TODO: refactor and deal with all this after file indexing - bvalfile = bids.internal.get_metadata(fullpath_filename, '^.*%s\\.bval$'); - if isfield(bvalfile, 'filename') - subject.dwi(end).dependencies.bval = bids.util.tsvread(bvalfile.filename); + subject.(modality)(end).metafile = bids.internal.get_meta_list(fullpath_filename); + subject.(modality)(end).dependencies.explicit = {}; + subject.(modality)(end).dependencies.data = {}; + subject.(modality)(end).dependencies.group = {}; + + ext = subject.(modality)(end).ext; + suffix = subject.(modality)(end).suffix; + search = strrep(file_list{i}, ['_' suffix ext], '_[a-zA-Z0-9.]+$'); + candidates = bids.internal.file_utils('List', pth, ['^' search '$']); + candidates = cellstr(candidates); + for ii = 1:numel(candidates) + if strcmp(candidates{ii}, file_list{i}) + continue; end - - bvecfile = bids.internal.get_metadata(fullpath_filename, '^.*%s\\.bvec$'); - if isfield(bvalfile, 'filename') - subject.dwi(end).dependencies.bvec = bids.util.tsvread(bvecfile.filename); + if bids.internal.endsWith(candidates{ii}, '.json') + continue + end + match = regexp(candidates{ii}, ['_' suffix '\..*$'], 'match'); + if isempty(match) % different suffix + subject.(modality)(end).dependencies.group{end+1} = fullfile(pth, candidates{ii}); + else % same suffix + subject.(modality)(end).dependencies.data{end+1} = fullfile(pth, candidates{ii}); end - end - end + end + end function subject = parse_perf(subject, schema) @@ -267,7 +243,32 @@ for i = 1:numel(file_list) + fullpath_filename = fullfile(pth, file_list{i}); subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema); + subject.(modality)(end).metafile = bids.internal.get_meta_list(fullpath_filename); + subject.(modality)(end).dependencies.explicit = {}; + subject.(modality)(end).dependencies.data = {}; + subject.(modality)(end).dependencies.group = {}; + + ext = subject.(modality)(end).ext; + suffix = subject.(modality)(end).suffix; + search = strrep(file_list{i}, ['_' suffix ext], '_[a-zA-Z0-9.]+$'); + candidates = bids.internal.file_utils('List', pth, ['^' search '$']); + candidates = cellstr(candidates); + for ii = 1:numel(candidates) + if strcmp(candidates{ii}, file_list{i}) + continue; + end + if bids.internal.endsWith(candidates{ii}, '.json') + continue + end + match = regexp(candidates{ii}, ['_' suffix '\..*$'], 'match'); + if isempty(match) % different suffix + subject.(modality)(end).dependencies.group{end+1} = fullfile(pth, candidates{ii}); + else % same suffix + subject.(modality)(end).dependencies.data{end+1} = fullfile(pth, candidates{ii}); + end + end switch subject.perf(i).suffix @@ -302,57 +303,6 @@ end -function subject = parse_fmap(subject, schema) - - modality = 'fmap'; - pth = fullfile(subject.path, modality); - - if exist(pth, 'dir') - - subject = bids.internal.add_missing_field(subject, modality); - - file_list = return_file_list(modality, subject, schema); - - for i = 1:numel(file_list) - - subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema); - - switch subject.fmap(i).suffix - - % -A single, real fieldmap image - case {'fieldmap', 'magnitude'} - subject.fmap(i).dependencies.magnitude = strrep(file_list{idx(i)}, ... - '_fieldmap.nii', ... - '_magnitude.nii'); - - % Phase difference image and at least one magnitude image - case {'phasediff'} - subject.fmap(i).dependencies.magnitude = { ... - strrep(file_list{i}, ... - '_phasediff.nii', ... - '_magnitude1.nii'), ... - strrep(file_list{i}, ... - '_phasediff.nii', ... - '_magnitude2.nii')}; % optional - - % Two phase images and two magnitude images - case {'phase1', 'phase2'} - subject.fmap(i).dependencies.magnitude = { ... - strrep(file_list{i}, ... - '_phase1.nii', ... - '_magnitude1.nii'), ... - strrep(file_list{i}, ... - '_phase1.nii', ... - '_magnitude2.nii')}; - - end - - end - - end - -end - % -------------------------------------------------------------------------- % HELPER FUNCTIONS % -------------------------------------------------------------------------- @@ -473,6 +423,34 @@ function tolerant_message(use_schema, msg) end +function BIDS = manage_dependencies(BIDS) + % Loops over all files and retrieve all files that current file depends on + file_list = bids.query(BIDS, 'data'); + for iFile = 1:size(file_list, 1) + info_src = bids.internal.return_file_info(BIDS, file_list{iFile}); + file = BIDS.subjects(info_src.sub_idx).(info_src.modality)(info_src.file_idx); + metadata = bids.internal.get_metadata(file.metafile); + + intended = {}; + if isfield(metadata, 'IntendedFor') + intended = cellstr(metadata.IntendedFor); + end + + for iIntended = 1:numel(intended) + dest = fullfile(BIDS.dir, BIDS.subjects(info_src.sub_idx).name, ... + intended{iIntended}); + if ~exist(dest, 'file') + warning(['IntendedFor file ' dest ' from ' file.filename ' not found']); + continue; + end + info_dest = bids.internal.return_file_info(BIDS, dest); + BIDS.subjects(info_dest.sub_idx).(info_dest.modality)(info_dest.file_idx)... + .dependencies.explicit{end+1} = file_list{iFile}; + end + + end +end + function BIDS = manage_intended_for(BIDS) % Loops through all the files with potential ``intentedFor`` metadata From 3b6ff80fc7167dbdb65d11380233cbf73f6566d0 Mon Sep 17 00:00:00 2001 From: Beliy Nikita Date: Wed, 24 Feb 2021 09:11:38 +0100 Subject: [PATCH 04/36] adapted query for metadata --- +bids/query.m | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/+bids/query.m b/+bids/query.m index 9cc7eb96..d6db3272 100644 --- a/+bids/query.m +++ b/+bids/query.m @@ -44,6 +44,7 @@ 'suffixes', ... 'data', ... 'metadata', ... + 'metafiles',... 'dependencies', ... 'extensions', ... 'prefixes'}; @@ -80,12 +81,14 @@ result = regexprep(result, '^[a-zA-Z0-9]+-', ''); result(cellfun('isempty', result)) = []; - case {'modalities', 'data'} + case {'modalities', 'data', 'metafiles'} result = result'; case {'metadata', 'dependencies'} if numel(result) == 1 result = result{1}; + else + result = result'; end case {'tasks', 'runs', 'suffixes', 'extensions', 'prefixes'} @@ -217,11 +220,16 @@ result{end + 1} = fullfile(BIDS.subjects(i).path, modalities{j}, d(k).filename); end - case 'metadata' + case 'metafiles' if isfield(d(k), 'filename') + fmeta = BIDS.subjects(i).(modalities{j})(k).metafile; + result = [result; fmeta]; + end - f = fullfile(BIDS.subjects(i).path, modalities{j}, d(k).filename); - result{end + 1, 1} = bids.internal.get_metadata(f); + case 'metadata' + if isfield(d(k), 'filename') + fmeta = BIDS.subjects(i).(modalities{j})(k).metafile; + result{end + 1, 1} = bids.internal.get_metadata(fmeta); if ~isempty(target) try result{end} = subsref(result{end}, target); From 0a541ca6d93e6b74cf9ab353b886fe4fd5691efd Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 27 Feb 2021 23:13:00 +0100 Subject: [PATCH 05/36] fix test and lint --- +bids/+internal/append_to_layout.m | 4 +- +bids/+internal/endsWith.m | 4 +- +bids/+internal/startsWith.m | 4 +- +bids/layout.m | 164 ++++++++++++++++------------- +bids/query.m | 2 +- tests/test_bids_query_asl.m | 53 ++++++---- tests/test_bids_query_dwi.m | 3 +- tests/test_bids_query_fmap.m | 5 +- tests/test_get_metadata.m | 7 +- 9 files changed, 134 insertions(+), 112 deletions(-) diff --git a/+bids/+internal/append_to_layout.m b/+bids/+internal/append_to_layout.m index 86715331..dd818ae9 100644 --- a/+bids/+internal/append_to_layout.m +++ b/+bids/+internal/append_to_layout.m @@ -1,4 +1,4 @@ -function subject = append_to_layout(file, subject, modality, schema) +function [subject, p] = append_to_layout(file, subject, modality, schema) % % appends a file to the BIDS layout by parsing it according to the provided schema % @@ -40,7 +40,7 @@ p = bids.internal.parse_filename(file, entities); % do not index json files when using the schema - if ~isempty(p) && strcmp(p.ext, '.json') + if isempty(p) || (~isempty(p) && strcmp(p.ext, '.json')) return end diff --git a/+bids/+internal/endsWith.m b/+bids/+internal/endsWith.m index a6cf7fc0..85f32692 100644 --- a/+bids/+internal/endsWith.m +++ b/+bids/+internal/endsWith.m @@ -16,8 +16,8 @@ res = false; l_pat = length(pat); if l_pat > length(str) - return; + return end - res = strcmp(str(end - l_pat +1 : end), pat); + res = strcmp(str(end - l_pat + 1:end), pat); end diff --git a/+bids/+internal/startsWith.m b/+bids/+internal/startsWith.m index 7cf84d17..ce98792b 100644 --- a/+bids/+internal/startsWith.m +++ b/+bids/+internal/startsWith.m @@ -16,8 +16,8 @@ res = false; l_pat = length(pat); if l_pat > length(str) - return; + return end - res = strcmp(str(1 : l_pat), pat); + res = strcmp(str(1:l_pat), pat); end diff --git a/+bids/layout.m b/+bids/layout.m index dc9f447c..5dc92d62 100644 --- a/+bids/layout.m +++ b/+bids/layout.m @@ -192,34 +192,45 @@ file_list = return_file_list(modality, subject, schema); - for i = 1:numel(file_list) - - fullpath_filename = fullfile(pth, file_list{i}); - subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema); - subject.(modality)(end).metafile = bids.internal.get_meta_list(fullpath_filename); - subject.(modality)(end).dependencies.explicit = {}; - subject.(modality)(end).dependencies.data = {}; - subject.(modality)(end).dependencies.group = {}; - - ext = subject.(modality)(end).ext; - suffix = subject.(modality)(end).suffix; - search = strrep(file_list{i}, ['_' suffix ext], '_[a-zA-Z0-9.]+$'); - candidates = bids.internal.file_utils('List', pth, ['^' search '$']); - candidates = cellstr(candidates); - for ii = 1:numel(candidates) - if strcmp(candidates{ii}, file_list{i}) - continue; - end - if bids.internal.endsWith(candidates{ii}, '.json') - continue - end - match = regexp(candidates{ii}, ['_' suffix '\..*$'], 'match'); - if isempty(match) % different suffix - subject.(modality)(end).dependencies.group{end+1} = fullfile(pth, candidates{ii}); - else % same suffix - subject.(modality)(end).dependencies.data{end+1} = fullfile(pth, candidates{ii}); + for iFile = 1:size(file_list, 1) + + fullpath_filename = fullfile(pth, file_list{iFile}); + [subject, parsing] = bids.internal.append_to_layout(file_list{iFile}, subject, modality, schema); + + if ~isempty(parsing) + + subject.(modality)(end).metafile = bids.internal.get_meta_list(fullpath_filename); + subject.(modality)(end).dependencies.explicit = {}; + subject.(modality)(end).dependencies.data = {}; + subject.(modality)(end).dependencies.group = {}; + + ext = subject.(modality)(end).ext; + suffix = subject.(modality)(end).suffix; + search = strrep(file_list{iFile}, ['_' suffix ext], '_[a-zA-Z0-9.]+$'); + candidates = bids.internal.file_utils('List', pth, ['^' search '$']); + candidates = cellstr(candidates); + + for ii = 1:numel(candidates) + + if strcmp(candidates{ii}, file_list{iFile}) + continue + end + + if bids.internal.endsWith(candidates{ii}, '.json') + continue + end + + match = regexp(candidates{ii}, ['_' suffix '\..*$'], 'match'); + if isempty(match) % different suffix + subject.(modality)(end).dependencies.group{end + 1, 1} = fullfile(pth, candidates{ii}); + else % same suffix + subject.(modality)(end).dependencies.data{end + 1, 1} = fullfile(pth, candidates{ii}); + end + end + end + end end @@ -241,59 +252,64 @@ file_list = return_file_list(modality, subject, schema); - for i = 1:numel(file_list) - - fullpath_filename = fullfile(pth, file_list{i}); - subject = bids.internal.append_to_layout(file_list{i}, subject, modality, schema); - subject.(modality)(end).metafile = bids.internal.get_meta_list(fullpath_filename); - subject.(modality)(end).dependencies.explicit = {}; - subject.(modality)(end).dependencies.data = {}; - subject.(modality)(end).dependencies.group = {}; - - ext = subject.(modality)(end).ext; - suffix = subject.(modality)(end).suffix; - search = strrep(file_list{i}, ['_' suffix ext], '_[a-zA-Z0-9.]+$'); - candidates = bids.internal.file_utils('List', pth, ['^' search '$']); - candidates = cellstr(candidates); - for ii = 1:numel(candidates) - if strcmp(candidates{ii}, file_list{i}) - continue; - end - if bids.internal.endsWith(candidates{ii}, '.json') - continue - end - match = regexp(candidates{ii}, ['_' suffix '\..*$'], 'match'); - if isempty(match) % different suffix - subject.(modality)(end).dependencies.group{end+1} = fullfile(pth, candidates{ii}); - else % same suffix - subject.(modality)(end).dependencies.data{end+1} = fullfile(pth, candidates{ii}); + for iFile = 1:numel(file_list) + + fullpath_filename = fullfile(pth, file_list{iFile}); + [subject, parsing] = bids.internal.append_to_layout(file_list{iFile}, subject, modality, schema); + + if ~isempty(parsing) + + subject.(modality)(end).metafile = bids.internal.get_meta_list(fullpath_filename); + subject.(modality)(end).dependencies.explicit = {}; + subject.(modality)(end).dependencies.data = {}; + subject.(modality)(end).dependencies.group = {}; + + ext = subject.(modality)(end).ext; + suffix = subject.(modality)(end).suffix; + search = strrep(file_list{iFile}, ['_' suffix ext], '_[a-zA-Z0-9.]+$'); + candidates = bids.internal.file_utils('List', pth, ['^' search '$']); + candidates = cellstr(candidates); + + for ii = 1:numel(candidates) + + if strcmp(candidates{ii}, file_list{iFile}) + continue + end + + if bids.internal.endsWith(candidates{ii}, '.json') + continue + end + + match = regexp(candidates{ii}, ['_' suffix '\..*$'], 'match'); + if isempty(match) % different suffix + subject.(modality)(end).dependencies.group{end + 1, 1} = fullfile(pth, candidates{ii}); + else % same suffix + subject.(modality)(end).dependencies.data{end + 1, 1} = fullfile(pth, candidates{ii}); + end + end - end - switch subject.perf(i).suffix + switch subject.(modality)(end).suffix - case 'asl' + case 'asl' - subject.perf(i).meta = []; - subject.perf(i).dependencies = []; + subject.(modality)(end).meta = []; - subject.perf(i).meta = bids.internal.get_metadata( ... - fullfile( ... - subject.path, ... - modality, ... - file_list{i})); + subject.(modality)(end).meta = bids.internal.get_metadata(subject.(modality)(iFile).metafile); - aslcontext_file = strrep(subject.perf(i).filename, ... - ['_asl' subject.perf(i).ext], ... - '_aslcontext.tsv'); - subject.perf(i).dependencies.context = manage_tsv( ... - struct('content', [], 'meta', []), ... - pth, ... - aslcontext_file); + aslcontext_file = strrep(subject.perf(end).filename, ... + ['_asl' subject.perf(end).ext], ... + '_aslcontext.tsv'); + subject.(modality)(end).dependencies.context = manage_tsv( ... + struct('content', [], 'meta', []), ... + pth, ... + aslcontext_file); - subject.perf(i) = manage_asllabeling(subject.perf(i), pth); + subject.(modality)(end) = manage_asllabeling(subject.perf(end), pth); - subject.perf(i) = manage_M0(subject.perf(i), pth); + subject.(modality)(end) = manage_M0(subject.perf(end), pth); + + end end @@ -441,11 +457,11 @@ function tolerant_message(use_schema, msg) intended{iIntended}); if ~exist(dest, 'file') warning(['IntendedFor file ' dest ' from ' file.filename ' not found']); - continue; + continue end info_dest = bids.internal.return_file_info(BIDS, dest); - BIDS.subjects(info_dest.sub_idx).(info_dest.modality)(info_dest.file_idx)... - .dependencies.explicit{end+1} = file_list{iFile}; + BIDS.subjects(info_dest.sub_idx).(info_dest.modality)(info_dest.file_idx) ... + .dependencies.explicit{end + 1, 1} = file_list{iFile}; end end diff --git a/+bids/query.m b/+bids/query.m index d6db3272..9f056d3f 100644 --- a/+bids/query.m +++ b/+bids/query.m @@ -44,7 +44,7 @@ 'suffixes', ... 'data', ... 'metadata', ... - 'metafiles',... + 'metafiles', ... 'dependencies', ... 'extensions', ... 'prefixes'}; diff --git a/tests/test_bids_query_asl.m b/tests/test_bids_query_asl.m index 4ed55f6e..77e346da 100644 --- a/tests/test_bids_query_asl.m +++ b/tests/test_bids_query_asl.m @@ -6,14 +6,30 @@ initTestSuite; end -function test_bids_query_asl_basic() - % - % asl queries - % +function test_bids_query_asl_basic_asl002() + + pth_bids_example = get_test_data_dir(); + + BIDS = bids.layout(fullfile(pth_bids_example, 'asl002')); + + modalities = {'anat', 'perf'}; + assertEqual(bids.query(BIDS, 'modalities'), modalities); + + suffixes = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'}; + assertEqual(bids.query(BIDS, 'suffixes'), suffixes); + + filename = bids.query(BIDS, 'data', 'sub', 'Sub103', 'suffix', 'm0scan'); + basename = bids.internal.file_utils(filename, 'basename'); + assertEqual(basename, {'sub-Sub103_m0scan.nii'}); + + assert(~isempty(BIDS.subjects.perf(1).dependencies.explicit)); + +end + +function test_bids_query_asl_basic_asl001() pth_bids_example = get_test_data_dir(); - %% 'asl001' BIDS = bids.layout(fullfile(pth_bids_example, 'asl001')); modalities = {'anat', 'perf'}; @@ -35,22 +51,12 @@ function test_bids_query_asl_basic() dependencies.context; dependencies.m0; - %% 'asl002' - BIDS = bids.layout(fullfile(pth_bids_example, 'asl002')); +end - modalities = {'anat', 'perf'}; - assertEqual(bids.query(BIDS, 'modalities'), modalities); +function test_bids_query_asl_basic_asl003() - suffixes = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'}; - assertEqual(bids.query(BIDS, 'suffixes'), suffixes); - - filename = bids.query(BIDS, 'data', 'sub', 'Sub103', 'suffix', 'm0scan'); - basename = bids.internal.file_utils(filename, 'basename'); - assertEqual(basename, {'sub-Sub103_m0scan.nii'}); - - assert(isfield(BIDS.subjects.perf(4), 'intended_for')); + pth_bids_example = get_test_data_dir(); - %% 'asl003' BIDS = bids.layout(fullfile(pth_bids_example, 'asl003')); modalities = {'anat', 'perf'}; @@ -59,7 +65,12 @@ function test_bids_query_asl_basic() suffixes = {'T1w', 'asl', 'aslcontext', 'asllabeling', 'm0scan'}; assertEqual(bids.query(BIDS, 'suffixes'), suffixes); - %% 'asl004' +end + +function test_bids_query_asl_basic_asl004() + + pth_bids_example = get_test_data_dir(); + BIDS = bids.layout(fullfile(pth_bids_example, 'asl004')); modalities = {'anat', 'fmap', 'perf'}; @@ -72,7 +83,7 @@ function test_bids_query_asl_basic() basename = bids.internal.file_utils(filename, 'basename'); assertEqual(basename, {'sub-Sub1_dir-pa_m0scan.nii'}); - assert(isfield(BIDS.subjects.fmap, 'intended_for')); - assert(isfield(BIDS.subjects.perf(4), 'intended_for')); + assert(~isempty(BIDS.subjects.perf(1).dependencies.explicit)); + assert(~isempty(BIDS.subjects.perf(4).dependencies)); end diff --git a/tests/test_bids_query_dwi.m b/tests/test_bids_query_dwi.m index 4807f2a0..29e5d4b4 100644 --- a/tests/test_bids_query_dwi.m +++ b/tests/test_bids_query_dwi.m @@ -28,6 +28,7 @@ function test_bids_query_dwi_basic() 'suffix', 'dwi', ... 'extension', '.nii.gz'); - assertEqual(dependencies.bval(1:11), [0 repmat(2400, 1, 10)]); + bval = bids.util.tsvread(dependencies.data{1}); + assertEqual(bval(1:11), [0 repmat(2400, 1, 10)]); end diff --git a/tests/test_bids_query_fmap.m b/tests/test_bids_query_fmap.m index dc49636e..0b48f700 100644 --- a/tests/test_bids_query_fmap.m +++ b/tests/test_bids_query_fmap.m @@ -13,11 +13,10 @@ function test_query_extension() BIDS = bids.layout(fullfile(pth_bids_example, '7t_trt')); - BIDS.subjects(1).fmap(3).intended_for; - BIDS.subjects(1).func(1).informed_by; + BIDS.subjects(1).func(1).dependencies.explicit; BIDS = bids.layout(fullfile(pth_bids_example, 'hcp_example_bids')); % sub-100307 - BIDS.subjects(1).fmap(3).intended_for{1}; + BIDS.subjects(1).anat(1).dependencies.explicit; end diff --git a/tests/test_get_metadata.m b/tests/test_get_metadata.m index e88278a2..5a208100 100644 --- a/tests/test_get_metadata.m +++ b/tests/test_get_metadata.m @@ -76,11 +76,6 @@ function test_get_metadata_internal() BIDS = bids.layout(fullfile(pth_bids_example, 'ds000117')); - bids.internal.get_metadata( ... - fullfile( ... - BIDS(1).subjects(2).path, ... - 'anat', ... - BIDS(1).subjects(2).anat(1).filename), ... - '%s'); + bids.internal.get_metadata(BIDS(1).subjects(2).anat(1).metafile); end From 59eb09e9c33be9bb2c4ba5a8be5068e122a99a5e Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 27 Feb 2021 23:18:26 +0100 Subject: [PATCH 06/36] remove dead code --- +bids/layout.m | 111 +------------------------------------------------ miss_hit.cfg | 4 +- 2 files changed, 3 insertions(+), 112 deletions(-) diff --git a/+bids/layout.m b/+bids/layout.m index 5dc92d62..b853a356 100644 --- a/+bids/layout.m +++ b/+bids/layout.m @@ -403,7 +403,7 @@ function tolerant_message(use_schema, msg) if strcmp(modality, 'meg') && ~isempty(d) for i = 1:size(d, 1) - file_list{end + 1, 1} = d(i, :); + file_list{end + 1, 1} = d(i, :); %#ok<*AGROW> end end @@ -467,115 +467,6 @@ function tolerant_message(use_schema, msg) end end -function BIDS = manage_intended_for(BIDS) - - % Loops through all the files with potential ``intentedFor`` metadata - % and creates an ``intended_for`` field with the fullpath list of all the target files - % it is intended for that exist. - % - % Also update the structure of each target file with an ``informed_by`` field - - suffix_with_intended_for = { ... - 'phasediff'; ... - 'phase1'; ... - 'phase2'; ... - 'fieldmap'; ... - 'epi'; ... - 'm0scan'; ... - 'coordsystem'}; - - for iSuffix = 1:numel(suffix_with_intended_for) - file_list = bids.query(BIDS, 'data', 'suffix', suffix_with_intended_for(iSuffix)); - - for iFile = 1:size(file_list, 1) - - info_src = bids.internal.return_file_info(BIDS, file_list{iFile}); - - [metadata, intended_for] = check_intended_for_exist(BIDS.subjects(info_src.sub_idx), ... - fullfile( ... - info_src.path, ... - info_src.modality, ... - info_src.filename)); - - BIDS.subjects(info_src.sub_idx).(info_src.modality)(info_src.file_idx).meta = metadata; - BIDS.subjects(info_src.sub_idx).(info_src.modality)(info_src.file_idx).intended_for = ... - intended_for; - - % update "dependency" field of target file - informed_by = []; - for iTargetFile = 1:size(intended_for, 1) - - info_tgt = bids.internal.return_file_info(BIDS, intended_for{iTargetFile}); - - % TODO: this should probably check that the informed_by field does not - % already exist in the structure - % TODO: This assumes that a file will only be informed by a single file from - % this moodality - informed_by.(info_src.modality) = file_list{iFile}; - BIDS.subjects(info_tgt.sub_idx).(info_tgt.modality)(info_tgt.file_idx).informed_by = ... - informed_by; - end - - end - - end - -end - -function [metadata, intended_for] = check_intended_for_exist(subject, filename) - - metadata = bids.internal.get_metadata(filename); - - intended_for = {}; - - if isempty(metadata) || ~isfield(metadata, 'IntendedFor') - warning('Missing field IntendedFor for %s', filename); - return - - else - - path_intended_for = {}; - - if ischar(metadata.IntendedFor) - path_intended_for{1, 1} = metadata.IntendedFor; - - elseif isstruct(metadata.IntendedFor) - for iPath = 1:length(metadata.IntendedFor) - path_intended_for{iPath, 1} = metadata.IntendedFor(iPath); %#ok<*AGROW> - end - - end - end - - subject_path = return_subject_path(subject); - - for iPath = 1:size(path_intended_for, 1) - - % create a fullname path for current operating system - fullpath_filename = fullfile(subject_path, ... - strrep(path_intended_for{iPath}, '/', filesep)); - - % check if this file is missing - if ~exist(fullpath_filename, 'file') - warning(['Missing: ' fullpath_filename]); - - else - intended_for{end + 1, 1} = fullpath_filename; - - end - end - -end - -function subject_path = return_subject_path(subject) - % get "subject path" without the session folder (if it exists) - subject_path = subject.path; - tmp = bids.internal.file_utils(subject_path, 'filename'); - if strcmp(tmp(1:3), 'ses') - subject_path = bids.internal.file_utils(subject_path, 'path'); - end -end - function perf = manage_asllabeling(perf, pth) % labeling image metadata (OPTIONAL) % --------------------------- diff --git a/miss_hit.cfg b/miss_hit.cfg index e418050c..58b9d5ce 100644 --- a/miss_hit.cfg +++ b/miss_hit.cfg @@ -29,5 +29,5 @@ tab_width: 2 # metrics limit for the code quality (https://florianschanda.github.io/miss_hit/metrics.html) metric "cnest": limit 6 metric "file_length": limit 600 -metric "cyc": limit 20 -metric "parameters": limit 8 \ No newline at end of file +metric "cyc": limit 18 +metric "parameters": limit 6 \ No newline at end of file From d502a8e7818bc0fce7e2e405a743aff0bb9a726e Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sun, 28 Feb 2021 09:37:19 +0100 Subject: [PATCH 07/36] minor fixes on get_metadata --- +bids/+internal/get_meta_list.m | 2 +- +bids/+internal/get_metadata.m | 7 +++++-- tests/test_get_metadata.m | 9 ++++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/+bids/+internal/get_meta_list.m b/+bids/+internal/get_meta_list.m index 5b28c617..cdee970e 100644 --- a/+bids/+internal/get_meta_list.m +++ b/+bids/+internal/get_meta_list.m @@ -71,7 +71,7 @@ % append path to list if ismeta - metalist{end + 1} = metafile{i}; + metalist{end + 1, 1} = metafile{i}; %#ok end end diff --git a/+bids/+internal/get_metadata.m b/+bids/+internal/get_metadata.m index cc8026b4..ad060d77 100644 --- a/+bids/+internal/get_metadata.m +++ b/+bids/+internal/get_metadata.m @@ -6,11 +6,14 @@ % % meta = bids.internal.get_metadata(metafile) % - % :param metafile: list of fullpath names of metafiles + % :param metafile: list of fullpath names of metafiles. % :type metafile: string or array of strings % + % :returns: - :meta: metadata structure % - % meta - metadata structure + % .. todo + % + % add explanation on how the inheritance principle is implemented. % __________________________________________________________________________ % Copyright (C) 2016-2018, Guillaume Flandin, Wellcome Centre for Human Neuroimaging diff --git a/tests/test_get_metadata.m b/tests/test_get_metadata.m index 5a208100..f6710f23 100644 --- a/tests/test_get_metadata.m +++ b/tests/test_get_metadata.m @@ -27,11 +27,9 @@ function test_get_metadata_basic() % define the expected output from bids query metadata func.RepetitionTime = 7; - - func_sub_01.RepetitionTime = 10; - anat.FlipAngle = 5; + func_sub_01.RepetitionTime = 10; anat_sub_01.FlipAngle = 10; anat_sub_01.Manufacturer = 'Siemens'; @@ -40,7 +38,8 @@ function test_get_metadata_basic() %% test func metadata base directory metadata = bids.query(BIDS, 'metadata', 'suffix', 'bold'); -% assert(metadata.RepetitionTime == func.RepetitionTime); + % assert(metadata.RepetitionTime == func.RepetitionTime); + %% test func metadata subject 01 metadata = bids.query(BIDS, 'metadata', 'sub', '01', 'suffix', 'bold'); @@ -48,7 +47,7 @@ function test_get_metadata_basic() %% test anat metadata base directory metadata = bids.query(BIDS, 'metadata', 'suffix', 'T1w'); -% assert(metadata.FlipAngle == anat.FlipAngle); + % assert(metadata.FlipAngle == anat.FlipAngle); %% test anat metadata subject 01 metadata = bids.query(BIDS, 'metadata', 'sub', '01', 'suffix', 'T1w'); From 2e0067297728bbe3f5bc61075111798681f46833 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sun, 28 Feb 2021 09:47:54 +0100 Subject: [PATCH 08/36] rename functions to use snake_case --- +bids/+internal/{endsWith.m => ends_with.m} | 6 +++--- +bids/+internal/get_metadata.m | 2 +- +bids/+internal/{startsWith.m => starts_with.m} | 6 +++--- +bids/layout.m | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) rename +bids/+internal/{endsWith.m => ends_with.m} (82%) rename +bids/+internal/{startsWith.m => starts_with.m} (83%) diff --git a/+bids/+internal/endsWith.m b/+bids/+internal/ends_with.m similarity index 82% rename from +bids/+internal/endsWith.m rename to +bids/+internal/ends_with.m index 85f32692..5233a3c9 100644 --- a/+bids/+internal/endsWith.m +++ b/+bids/+internal/ends_with.m @@ -1,4 +1,4 @@ -function res = endsWith(str, pat) +function res = ends_with(str, pattern) % % Checks id character array 'str' ends with 'pat' % @@ -14,10 +14,10 @@ % Copyright (C) 2011-2018 Guillaume Flandin, Wellcome Centre for Human Neuroimaging res = false; - l_pat = length(pat); + l_pat = length(pattern); if l_pat > length(str) return end - res = strcmp(str(end - l_pat + 1:end), pat); + res = strcmp(str(end - l_pat + 1:end), pattern); end diff --git a/+bids/+internal/get_metadata.m b/+bids/+internal/get_metadata.m index ad060d77..14542a83 100644 --- a/+bids/+internal/get_metadata.m +++ b/+bids/+internal/get_metadata.m @@ -23,7 +23,7 @@ metafile = cellstr(metafile); for i = 1:numel(metafile) - if bids.internal.endsWith(metafile{i}, '.json') + if bids.internal.ends_with(metafile{i}, '.json') meta = update_metadata(meta, bids.util.jsondecode(metafile{i}), metafile{i}); else meta.filename = metafile{i}; diff --git a/+bids/+internal/startsWith.m b/+bids/+internal/starts_with.m similarity index 83% rename from +bids/+internal/startsWith.m rename to +bids/+internal/starts_with.m index ce98792b..272be3d8 100644 --- a/+bids/+internal/startsWith.m +++ b/+bids/+internal/starts_with.m @@ -1,4 +1,4 @@ -function res = startsWith(str, pat) +function res = starts_with(str, pattern) % % Checks id character array 'str' starts with 'pat' % @@ -14,10 +14,10 @@ % Copyright (C) 2011-2018 Guillaume Flandin, Wellcome Centre for Human Neuroimaging res = false; - l_pat = length(pat); + l_pat = length(pattern); if l_pat > length(str) return end - res = strcmp(str(1:l_pat), pat); + res = strcmp(str(1:l_pat), pattern); end diff --git a/+bids/layout.m b/+bids/layout.m index b853a356..879e67ec 100644 --- a/+bids/layout.m +++ b/+bids/layout.m @@ -216,7 +216,7 @@ continue end - if bids.internal.endsWith(candidates{ii}, '.json') + if bids.internal.ends_with(candidates{ii}, '.json') continue end @@ -276,7 +276,7 @@ continue end - if bids.internal.endsWith(candidates{ii}, '.json') + if bids.internal.ends_with(candidates{ii}, '.json') continue end From e20e967918a8db57cd5a045cbabe206b15f915a0 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sun, 28 Feb 2021 12:46:44 +0100 Subject: [PATCH 09/36] refactor dependency indexing --- +bids/layout.m | 70 ++++++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/+bids/layout.m b/+bids/layout.m index 879e67ec..ecfc7af4 100644 --- a/+bids/layout.m +++ b/+bids/layout.m @@ -194,40 +194,11 @@ for iFile = 1:size(file_list, 1) - fullpath_filename = fullfile(pth, file_list{iFile}); [subject, parsing] = bids.internal.append_to_layout(file_list{iFile}, subject, modality, schema); if ~isempty(parsing) - subject.(modality)(end).metafile = bids.internal.get_meta_list(fullpath_filename); - subject.(modality)(end).dependencies.explicit = {}; - subject.(modality)(end).dependencies.data = {}; - subject.(modality)(end).dependencies.group = {}; - - ext = subject.(modality)(end).ext; - suffix = subject.(modality)(end).suffix; - search = strrep(file_list{iFile}, ['_' suffix ext], '_[a-zA-Z0-9.]+$'); - candidates = bids.internal.file_utils('List', pth, ['^' search '$']); - candidates = cellstr(candidates); - - for ii = 1:numel(candidates) - - if strcmp(candidates{ii}, file_list{iFile}) - continue - end - - if bids.internal.ends_with(candidates{ii}, '.json') - continue - end - - match = regexp(candidates{ii}, ['_' suffix '\..*$'], 'match'); - if isempty(match) % different suffix - subject.(modality)(end).dependencies.group{end + 1, 1} = fullfile(pth, candidates{ii}); - else % same suffix - subject.(modality)(end).dependencies.data{end + 1, 1} = fullfile(pth, candidates{ii}); - end - - end + subject = index_dependencies(subject, modality, file_list{iFile}); end @@ -409,6 +380,45 @@ function tolerant_message(use_schema, msg) end +function subject = index_dependencies(subject, modality, file) + + pth = fullfile(subject.path, modality); + fullpath_filename = fullfile(pth, file); + + subject.(modality)(end).metafile = bids.internal.get_meta_list(fullpath_filename); + subject.(modality)(end).dependencies.explicit = {}; + subject.(modality)(end).dependencies.data = {}; + subject.(modality)(end).dependencies.group = {}; + + ext = subject.(modality)(end).ext; + suffix = subject.(modality)(end).suffix; + pattern = strrep(file, ['_' suffix ext], '_[a-zA-Z0-9.]+$'); + candidates = bids.internal.file_utils('List', pth, ['^' pattern '$']); + candidates = cellstr(candidates); + + for ii = 1:numel(candidates) + + if strcmp(candidates{ii}, file) + continue + end + + if bids.internal.ends_with(candidates{ii}, '.json') + continue + end + + match = regexp(candidates{ii}, ['_' suffix '\..*$'], 'match'); + % different suffix + if isempty(match) + subject.(modality)(end).dependencies.group{end + 1, 1} = fullfile(pth, candidates{ii}); + % same suffix + else + subject.(modality)(end).dependencies.data{end + 1, 1} = fullfile(pth, candidates{ii}); + end + + end + +end + function structure = manage_tsv(structure, pth, filename) % Retunrs the content and metadata of a TSV file (if they exist) % From e0d12ba8088e21705d0b838c24faa471eaa6f897 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sun, 28 Feb 2021 12:49:13 +0100 Subject: [PATCH 10/36] get rid of parse_perf --- +bids/layout.m | 64 +------------------------------------------------- 1 file changed, 1 insertion(+), 63 deletions(-) diff --git a/+bids/layout.m b/+bids/layout.m index ecfc7af4..f9d681bd 100644 --- a/+bids/layout.m +++ b/+bids/layout.m @@ -166,10 +166,8 @@ % so the parsing is unconstrained for iModality = 1:numel(modalities) switch modalities{iModality} - case {'anat', 'func', 'beh', 'meg', 'eeg', 'ieeg', 'pet', 'fmap', 'dwi'} + case {'anat', 'func', 'beh', 'meg', 'eeg', 'ieeg', 'pet', 'fmap', 'dwi', 'perf'} subject = parse_using_schema(subject, modalities{iModality}, schema); - case 'perf' - subject = parse_perf(subject, schema); otherwise % in case we are going schemaless % and the modality is not one of the usual suspect @@ -200,66 +198,6 @@ subject = index_dependencies(subject, modality, file_list{iFile}); - end - - end - - end - -end - -function subject = parse_perf(subject, schema) - - % -------------------------------------------------------------------------- - % ASL perfusion imaging data - % -------------------------------------------------------------------------- - - modality = 'perf'; - pth = fullfile(subject.path, 'perf'); - - if exist(pth, 'dir') - - subject = bids.internal.add_missing_field(subject, modality); - - file_list = return_file_list(modality, subject, schema); - - for iFile = 1:numel(file_list) - - fullpath_filename = fullfile(pth, file_list{iFile}); - [subject, parsing] = bids.internal.append_to_layout(file_list{iFile}, subject, modality, schema); - - if ~isempty(parsing) - - subject.(modality)(end).metafile = bids.internal.get_meta_list(fullpath_filename); - subject.(modality)(end).dependencies.explicit = {}; - subject.(modality)(end).dependencies.data = {}; - subject.(modality)(end).dependencies.group = {}; - - ext = subject.(modality)(end).ext; - suffix = subject.(modality)(end).suffix; - search = strrep(file_list{iFile}, ['_' suffix ext], '_[a-zA-Z0-9.]+$'); - candidates = bids.internal.file_utils('List', pth, ['^' search '$']); - candidates = cellstr(candidates); - - for ii = 1:numel(candidates) - - if strcmp(candidates{ii}, file_list{iFile}) - continue - end - - if bids.internal.ends_with(candidates{ii}, '.json') - continue - end - - match = regexp(candidates{ii}, ['_' suffix '\..*$'], 'match'); - if isempty(match) % different suffix - subject.(modality)(end).dependencies.group{end + 1, 1} = fullfile(pth, candidates{ii}); - else % same suffix - subject.(modality)(end).dependencies.data{end + 1, 1} = fullfile(pth, candidates{ii}); - end - - end - switch subject.(modality)(end).suffix case 'asl' From 698251d622f4eb5841dbd9844d7e9e13235059a9 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sun, 28 Feb 2021 12:51:46 +0100 Subject: [PATCH 11/36] remove commented out dead code --- +bids/layout.m | 1 - 1 file changed, 1 deletion(-) diff --git a/+bids/layout.m b/+bids/layout.m index f9d681bd..744268cc 100644 --- a/+bids/layout.m +++ b/+bids/layout.m @@ -125,7 +125,6 @@ %% Dependencies % ========================================================================== - % BIDS = manage_intended_for(BIDS); BIDS = manage_dependencies(BIDS); end From e51ac08d2835089cad154fe74950f25e9bf33949 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sun, 28 Feb 2021 13:05:56 +0100 Subject: [PATCH 12/36] add comments --- +bids/layout.m | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/+bids/layout.m b/+bids/layout.m index 744268cc..e9de39ff 100644 --- a/+bids/layout.m +++ b/+bids/layout.m @@ -291,8 +291,8 @@ function tolerant_message(use_schema, msg) % TODO % this does not cover coordsystem.json - % jn to omit json but not .pos file for headshape.pos + % jn to omit json but not .pos file for headshape.pos pattern = '_([a-zA-Z0-9]+){1}\\..*[^jn]'; prefix = ''; if isempty(schema) @@ -318,6 +318,20 @@ function tolerant_message(use_schema, msg) end function subject = index_dependencies(subject, modality, file) + % + % Each file structure contains dependencies sub-structure with guaranteed fields: + % + % - explicit: list of data files containing "IntendedFor" referencing current file. + % see the manage_dependencies function + % + % - data: list of files with same name but different extension. + % This combines files that are split in header and data + % (like in Brainvision), also takes care of bval/bvec files + % + % - group: list of files that have same name except extension and suffix. + % This groups file that logically need each other, + % like functional mri and events tabular file. + % It also takes care of fmap magnitude1/2 and phasediff. pth = fullfile(subject.path, modality); fullpath_filename = fullfile(pth, file); @@ -357,12 +371,11 @@ function tolerant_message(use_schema, msg) end function structure = manage_tsv(structure, pth, filename) - % Retunrs the content and metadata of a TSV file (if they exist) + % Returns the content and metadata of a TSV file (if they exist) % % NOTE: inheritance principle not implemented. % Does NOT look for the metadata of a file at higher levels % - % ext = bids.internal.file_utils(filename, 'ext'); tsv_file = bids.internal.file_utils('FPList', ... @@ -387,13 +400,24 @@ function tolerant_message(use_schema, msg) end function BIDS = manage_dependencies(BIDS) + % % Loops over all files and retrieve all files that current file depends on + % + file_list = bids.query(BIDS, 'data'); + for iFile = 1:size(file_list, 1) + info_src = bids.internal.return_file_info(BIDS, file_list{iFile}); file = BIDS.subjects(info_src.sub_idx).(info_src.modality)(info_src.file_idx); metadata = bids.internal.get_metadata(file.metafile); + % If the file A is intended for file B + % then we update the dependencies.explicit field structrure of file B + % so it contains the fullpath to file A + % + % This way when one queries info about B, then it is easy to know what + % other is present to help with analysis. intended = {}; if isfield(metadata, 'IntendedFor') intended = cellstr(metadata.IntendedFor); @@ -412,11 +436,12 @@ function tolerant_message(use_schema, msg) end end + end function perf = manage_asllabeling(perf, pth) % labeling image metadata (OPTIONAL) - % --------------------------- + metafile = fullfile(pth, strrep(perf.filename, ... ['_asl' perf.ext], ... '_asllabeling.jpg')); @@ -430,7 +455,6 @@ function tolerant_message(use_schema, msg) end function perf = manage_M0(perf, pth) - % M0 field is flexible: if ~isfield(perf.meta, 'M0Type') From 28e7ee02854639dacbc4894e63c8f99fa5d9cfab Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 16 Apr 2021 16:22:26 +0200 Subject: [PATCH 13/36] remove extra IF block --- +bids/+internal/get_metadata.m | 2 +- +bids/query.m | 38 +++++++++++++--------------------- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/+bids/+internal/get_metadata.m b/+bids/+internal/get_metadata.m index 14542a83..b732cafc 100644 --- a/+bids/+internal/get_metadata.m +++ b/+bids/+internal/get_metadata.m @@ -6,7 +6,7 @@ % % meta = bids.internal.get_metadata(metafile) % - % :param metafile: list of fullpath names of metafiles. + % :param metafile: list of fullpath names of metadata files. % :type metafile: string or array of strings % % :returns: - :meta: metadata structure diff --git a/+bids/query.m b/+bids/query.m index 9f056d3f..0e521451 100644 --- a/+bids/query.m +++ b/+bids/query.m @@ -216,30 +216,24 @@ result = union(result, allmods(hasmod)); case 'data' - if isfield(d(k), 'filename') - result{end + 1} = fullfile(BIDS.subjects(i).path, modalities{j}, d(k).filename); - end + result{end + 1} = fullfile(BIDS.subjects(i).path, modalities{j}, d(k).filename); case 'metafiles' - if isfield(d(k), 'filename') - fmeta = BIDS.subjects(i).(modalities{j})(k).metafile; - result = [result; fmeta]; - end + fmeta = BIDS.subjects(i).(modalities{j})(k).metafile; + result = [result; fmeta]; case 'metadata' - if isfield(d(k), 'filename') - fmeta = BIDS.subjects(i).(modalities{j})(k).metafile; - result{end + 1, 1} = bids.internal.get_metadata(fmeta); - if ~isempty(target) - try - result{end} = subsref(result{end}, target); - catch - warning('Non-existent field for metadata.'); - result{end} = []; - end + fmeta = BIDS.subjects(i).(modalities{j})(k).metafile; + result{end + 1, 1} = bids.internal.get_metadata(fmeta); + if ~isempty(target) + try + result{end} = subsref(result{end}, target); + catch + warning('Non-existent field for metadata.'); + result{end} = []; end - end + % if status && isfield(d(k),'meta') % result{end+1} = d(k).meta; % end @@ -255,9 +249,7 @@ end case 'suffixes' - if isfield(d(k), 'suffix') - result{end + 1} = d(k).suffix; - end + result{end + 1} = d(k).suffix; case 'prefixes' if isfield(d(k), 'prefix') @@ -265,9 +257,7 @@ end case 'extensions' - if isfield(d(k), 'ext') - result{end + 1} = d(k).ext; - end + result{end + 1} = d(k).ext; case 'dependencies' if isfield(d(k), 'dependencies') From e41e762946e49ca835d46875e0de716c97d378ad Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 16 Apr 2021 16:34:09 +0200 Subject: [PATCH 14/36] ASL jpg file handled like other dependency files --- +bids/layout.m | 17 ----------------- tests/test_bids_query_asl.m | 4 +++- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/+bids/layout.m b/+bids/layout.m index e9de39ff..de84c938 100644 --- a/+bids/layout.m +++ b/+bids/layout.m @@ -213,8 +213,6 @@ pth, ... aslcontext_file); - subject.(modality)(end) = manage_asllabeling(subject.perf(end), pth); - subject.(modality)(end) = manage_M0(subject.perf(end), pth); end @@ -439,21 +437,6 @@ function tolerant_message(use_schema, msg) end -function perf = manage_asllabeling(perf, pth) - % labeling image metadata (OPTIONAL) - - metafile = fullfile(pth, strrep(perf.filename, ... - ['_asl' perf.ext], ... - '_asllabeling.jpg')); - - if exist(metafile, 'file') - [~, Ffile] = fileparts(metafile); - perf.dependencies.labeling_image.filename = [Ffile '.jpg']; - - end - -end - function perf = manage_M0(perf, pth) % M0 field is flexible: diff --git a/tests/test_bids_query_asl.m b/tests/test_bids_query_asl.m index 77e346da..89df841f 100644 --- a/tests/test_bids_query_asl.m +++ b/tests/test_bids_query_asl.m @@ -47,7 +47,9 @@ function test_bids_query_asl_basic_asl001() meta = bids.query(BIDS, 'metadata', 'sub', 'Sub103', 'suffix', 'asl'); dependencies = bids.query(BIDS, 'dependencies', 'sub', 'Sub103', 'suffix', 'asl'); - assertEqual(dependencies.labeling_image.filename, 'sub-Sub103_asllabeling.jpg'); + assert(any(ismember( ... + bids.internal.file_utils(dependencies.group, 'filename'), ... + 'sub-Sub103_asllabeling.jpg'))); dependencies.context; dependencies.m0; From d26c10431244a5b5139e9f0a5c415d1933f24330 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 16 Apr 2021 16:55:38 +0200 Subject: [PATCH 15/36] update ASL query tests to make sure context file dependency is indexed --- tests/test_bids_query_asl.m | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_bids_query_asl.m b/tests/test_bids_query_asl.m index 89df841f..8fff5e4f 100644 --- a/tests/test_bids_query_asl.m +++ b/tests/test_bids_query_asl.m @@ -24,6 +24,11 @@ function test_bids_query_asl_basic_asl002() assert(~isempty(BIDS.subjects.perf(1).dependencies.explicit)); + dependencies = bids.query(BIDS, 'dependencies', 'sub', 'Sub103', 'suffix', 'asl'); + assert(any(ismember( ... + bids.internal.file_utils(dependencies.group, 'filename'), ... + 'sub-Sub103_aslcontext.tsv'))); + end function test_bids_query_asl_basic_asl001() From 7352e6b04e9d841af427577c8a5c9debf453265d Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 16 Apr 2021 19:49:39 +0200 Subject: [PATCH 16/36] improving parsing of prefixes --- +bids/+internal/parse_filename.m | 12 ++++++++++-- tests/test_parse_filename.m | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/+bids/+internal/parse_filename.m b/+bids/+internal/parse_filename.m index 29927797..cea3ad97 100644 --- a/+bids/+internal/parse_filename.m +++ b/+bids/+internal/parse_filename.m @@ -53,8 +53,16 @@ end % identidy an eventual prefix to the file - tmp = regexp(parts{1}, '(sub)', 'split'); - p.prefix = tmp{1}; + % and amends the sub entity accordingly + p.prefix = ''; + if strfind(parts{1}, 'sub-') + tmp = regexp(parts{1}, '(sub-)', 'split'); + p.prefix = tmp{1}; + if ~isempty(p.prefix) + p.entities.sub = p.entities.([p.prefix 'sub']); + p.entities = rmfield(p.entities, [p.prefix 'sub']); + end + end % -Extra fields can be added to the structure and ordered specifically. if nargin == 2 diff --git a/tests/test_parse_filename.m b/tests/test_parse_filename.m index 6b658d1d..18ed1ea2 100644 --- a/tests/test_parse_filename.m +++ b/tests/test_parse_filename.m @@ -16,7 +16,7 @@ function test_parse_filename_prefix() 'suffix', 'bold', ... 'prefix', 'swua', ... 'ext', '.nii', ... - 'entities', struct('swuasub', '16', ... + 'entities', struct('sub', '16', ... 'run', '1', ... 'task', 'rest')); From 2ab00b69978377809e567fb2613cbb8084c5d3a8 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 16 Apr 2021 19:53:32 +0200 Subject: [PATCH 17/36] silence get_metadata tests their reactivation will require a public function to index and read associated metadata --- tests/test_get_metadata.m | 33 +++++++++++++++++------------- tests/test_get_metadata_suffixes.m | 33 ++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 23 deletions(-) diff --git a/tests/test_get_metadata.m b/tests/test_get_metadata.m index f6710f23..50b31ec7 100644 --- a/tests/test_get_metadata.m +++ b/tests/test_get_metadata.m @@ -6,6 +6,25 @@ initTestSuite; end +function test_get_metadata_participants() + % test files with no underscore in name. + + pth_bids_example = get_test_data_dir(); + + file = fullfile(pth_bids_example, 'ds001', 'participants.tsv'); + side_car = fullfile(pth_bids_example, 'ds001', 'participants.json'); + + % SILENCING TEST + % bids.internal.get_metadata now only takes .jsons as input + % bids.internal.get_meta_list is now in charge of building the list of + % metadata file list + + % metadata = bids.internal.get_metadata(file); + % expected_metadata = bids.util.jsondecode(side_car); + % assertEqual(metadata, expected_metadata); + +end + function test_get_metadata_basic() % Test metadata and the inheritance principle % __________________________________________________________________________ @@ -40,7 +59,6 @@ function test_get_metadata_basic() metadata = bids.query(BIDS, 'metadata', 'suffix', 'bold'); % assert(metadata.RepetitionTime == func.RepetitionTime); - %% test func metadata subject 01 metadata = bids.query(BIDS, 'metadata', 'sub', '01', 'suffix', 'bold'); assert(metadata.RepetitionTime == func_sub_01.RepetitionTime); @@ -56,19 +74,6 @@ function test_get_metadata_basic() end -function test_get_metadata_participants() - % test files with no underscore in name. - - pth_bids_example = get_test_data_dir(); - - file = fullfile(pth_bids_example, 'ds001', 'participants.tsv'); - side_car = fullfile(pth_bids_example, 'ds001', 'participants.json'); - metadata = bids.internal.get_metadata(file); - expected_metadata = bids.util.jsondecode(side_car); - assertEqual(metadata, expected_metadata); - -end - function test_get_metadata_internal() pth_bids_example = get_test_data_dir(); diff --git a/tests/test_get_metadata_suffixes.m b/tests/test_get_metadata_suffixes.m index 9890bdb9..e43f36c3 100644 --- a/tests/test_get_metadata_suffixes.m +++ b/tests/test_get_metadata_suffixes.m @@ -13,20 +13,35 @@ function test_get_metadata_suffixes_basic() file = fullfile(data_dir, 'sub-06_hemi-R_space-individual_den-native_thickness.shape.gii'); side_car = fullfile(data_dir, 'sub-06_hemi-R_space-individual_den-native_thickness.json'); - metadata = bids.internal.get_metadata(file); - expected_metadata = bids.util.jsondecode(side_car); - assertEqual(metadata, expected_metadata); + + % SILENCING TEST + % bids.internal.get_metadata now only takes .jsons as input + % bids.internal.get_meta_list is now in charge of building the list of + % metadata file list + % metadata = bids.internal.get_metadata(file); + % expected_metadata = bids.util.jsondecode(side_car); + % assertEqual(metadata, expected_metadata); file = fullfile(data_dir, 'sub-06_hemi-R_space-individual_den-native_midthickness.surf.gii'); side_car = fullfile(data_dir, 'sub-06_hemi-R_space-individual_den-native_midthickness.json'); - metadata = bids.internal.get_metadata(file); - expected_metadata = bids.util.jsondecode(side_car); - assertEqual(metadata, expected_metadata); + + % SILENCING TEST + % bids.internal.get_metadata now only takes .jsons as input + % bids.internal.get_meta_list is now in charge of building the list of + % metadata file list + % metadata = bids.internal.get_metadata(file); + % expected_metadata = bids.util.jsondecode(side_car); + % assertEqual(metadata, expected_metadata); file = fullfile(data_dir, 'sub-06_space-individual_den-native_thickness.dscalar.nii'); side_car = fullfile(data_dir, 'sub-06_space-individual_den-native_thickness.json'); - metadata = bids.internal.get_metadata(file); - expected_metadata = bids.util.jsondecode(side_car); - assertEqual(metadata, expected_metadata); + + % SILENCING TEST + % bids.internal.get_metadata now only takes .jsons as input + % bids.internal.get_meta_list is now in charge of building the list of + % metadata file list + % metadata = bids.internal.get_metadata(file); + % expected_metadata = bids.util.jsondecode(side_car); + % assertEqual(metadata, expected_metadata); end From ed237cadc720a6b7b1ae0ba72f35f7e676a05fe8 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 16 Apr 2021 19:54:56 +0200 Subject: [PATCH 18/36] readd subfunction deleted during rebase --- +bids/layout.m | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/+bids/layout.m b/+bids/layout.m index de84c938..b4906050 100644 --- a/+bids/layout.m +++ b/+bids/layout.m @@ -152,8 +152,8 @@ % NOTE: *_scans.json files can stored at the root level % and this should implemented when querying scans.tsv content + metadata subject.scans = bids.internal.file_utils('FPList', ... - subject.path, ... - ['^' subjname, '.*_scans.tsv' '$']); + subject.path, ... + ['^' subjname, '.*_scans.tsv' '$']); modality_groups = bids.schema.return_modality_groups(schema); @@ -279,6 +279,15 @@ function tolerant_message(use_schema, msg) end end +function subject_path = return_subject_path(subject) + % get "subject path" without the session folder (if it exists) + subject_path = subject.path; + tmp = bids.internal.file_utils(subject_path, 'filename'); + if strcmp(tmp(1:3), 'ses') + subject_path = bids.internal.file_utils(subject_path, 'path'); + end +end + function file_list = return_file_list(modality, subject, schema) % We list anything but json files From 1c6e897bb811de4595bb8d454d021512df4091c7 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 16 Apr 2021 19:56:55 +0200 Subject: [PATCH 19/36] reimplement #195 --- +bids/+internal/get_meta_list.m | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/+bids/+internal/get_meta_list.m b/+bids/+internal/get_meta_list.m index cdee970e..c5948cf4 100644 --- a/+bids/+internal/get_meta_list.m +++ b/+bids/+internal/get_meta_list.m @@ -58,9 +58,14 @@ entities = fieldnames(p2.entities); end - % -Check if this metadata file contains the same entity-label pairs as its - % data file counterpart + % Check if this metadata file contains + % - the same entity-label pairs + % - same suffix + % as its data file counterpart ismeta = true; + if ~strcmp(p.suffix, p2.suffix) + ismeta = false; + end for j = 1:numel(entities) if ~isfield(p.entities, entities{j}) || ... ~strcmp(p.entities.(entities{j}), p2.entities.(entities{j})) From 2fb7b157b4d93b5392dfa2c70c8efb35d1058e4e Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 16 Apr 2021 19:57:14 +0200 Subject: [PATCH 20/36] linting --- +bids/derivate.m | 18 +++++++------- tests/test_bids_query_sessions_scans_tsv.m | 1 - tests/test_layout_derivatives.m | 2 +- tests/test_tsvwrite.m | 28 +++++++++++----------- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/+bids/derivate.m b/+bids/derivate.m index 524389ec..05f8110d 100644 --- a/+bids/derivate.m +++ b/+bids/derivate.m @@ -40,13 +40,13 @@ if isempty(data_list) warning(['No data found for this query']); - return; + return else fprintf('Found %d files in %d subjects\n', length(data_list), length(subjects_list)); end pth_BIDSderiv = fullfile(out_path, name); - if ~exist(pth_BIDSderiv,'dir') + if ~exist(pth_BIDSderiv, 'dir') mkdir(pth_BIDSderiv); end @@ -86,15 +86,15 @@ info = bids.internal.return_file_info(BIDS, data_file); file = BIDS.subjects(info.sub_idx).(info.modality)(info.file_idx); basename = file.filename(1:end - length(file.ext)); - out_dir = fullfile(derivatives_folder,... - BIDS.subjects(info.sub_idx).name,... - BIDS.subjects(info.sub_idx).session,... + out_dir = fullfile(derivatives_folder, ... + BIDS.subjects(info.sub_idx).name, ... + BIDS.subjects(info.sub_idx).session, ... info.modality); out_path = fullfile(out_dir, basename); meta_file = [out_path '.json']; % ignore already existing files; avoid circular references - if exist(meta_file) + if exist(meta_file) return end if ~exist(out_dir, 'dir') @@ -104,11 +104,11 @@ if endsWith(file.ext, '.gz') gunzip(data_file, out_dir); else - [status,message,messageId] = copyfile(data_file, [out_path file.ext]); + [status, message, messageId] = copyfile(data_file, [out_path file.ext]); end if ~status warning([messageId ': ' message]); - return; + return end % export metadata if ~strcmpi(file.ext, '.json') % skip if data file is json @@ -124,7 +124,7 @@ if ~isempty(file.dependencies) dependencies = fieldnames(file.dependencies); for dep = 1:numel(dependencies) - for idep = 1: numel(file.dependencies.(dependencies{dep})) + for idep = 1:numel(file.dependencies.(dependencies{dep})) dep_file = file.dependencies.(dependencies{dep}){idep}; if exist(dep_file, 'file') copy_file(BIDS, derivatives_folder, dep_file); diff --git a/tests/test_bids_query_sessions_scans_tsv.m b/tests/test_bids_query_sessions_scans_tsv.m index 3154ce24..3f9194a9 100644 --- a/tests/test_bids_query_sessions_scans_tsv.m +++ b/tests/test_bids_query_sessions_scans_tsv.m @@ -19,7 +19,6 @@ function test_query_sessions_tsv() assert(~isempty(BIDS.subjects(1).sess)); assert(~isempty(BIDS.subjects(1).scans)); - end function test_query_scans_tsv() diff --git a/tests/test_layout_derivatives.m b/tests/test_layout_derivatives.m index 092bddbd..a1b2bdda 100644 --- a/tests/test_layout_derivatives.m +++ b/tests/test_layout_derivatives.m @@ -25,7 +25,7 @@ function test_layout_prefix() 'prefix', 'swua'); basename = bids.internal.file_utils(data, 'basename'); assertEqual(basename, {'swuasub-01_task-balloonanalogrisktask_run-01_bold'}); - + assertEqual(bids.query(BIDS, 'prefixes'), {'swua'}); end diff --git a/tests/test_tsvwrite.m b/tests/test_tsvwrite.m index b557df8d..fb7b7295 100644 --- a/tests/test_tsvwrite.m +++ b/tests/test_tsvwrite.m @@ -99,44 +99,44 @@ function test_tsvwrite_basic() % as a possible data shape for the input opf TSV write % function test_tsvwrite_row_wise_structure() -% +% % pth = fileparts(mfilename('fullpath')); -% +% % tsv_file = fullfile(pth, 'sub-01_task-STRUCTURE_events.tsv'); -% +% % logFile(1, 1).onset = 2; % logFile(1, 1).trial_type = 'motion_up'; % logFile(1, 1).duration = 1; % logFile(1, 1).speed = []; % logFile(1, 1).is_fixation = true; -% +% % logFile(2, 1).onset = NaN; % logFile(2, 1).trial_type = 'static'; % logFile(2, 1).duration = 4; % logFile(2, 1).speed = 4; % logFile(2, 1).is_fixation = 3; -% +% % bids.util.tsvwrite(tsv_file, logFile); -% +% % % read the file & % % check the extra columns of the header and some of the content -% +% % FID = fopen(tsv_file, 'r'); % C = textscan(FID, '%s%s%s%s%s', 'Delimiter', '\t', 'EndOfLine', '\n'); -% +% % % check header % assertEqual(C{4}{1}, 'speed'); -% +% % % check that empty values are entered as NaN: logFile.speed(1) % assertEqual(C{4}{2}, 'n/a'); -% +% % % check that missing fields are entered as NaN: logFile.speed(2) % assertEqual(C{4}{3}, '4'); -% +% % % check that NaN are written as : logFile.onset(2) % assertEqual(C{1}{3}, 'n/a'); % -% +% % % check values entered properly: logFile.is_fixation(2) % assertEqual(C{5}{3}, '3'); -% -% end \ No newline at end of file +% +% end From a63de7c59356e0bb47e219e0822a0eab4dfafb90 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 16 Apr 2021 21:09:37 +0200 Subject: [PATCH 21/36] linting --- +bids/layout.m | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/+bids/layout.m b/+bids/layout.m index b4906050..87267a38 100644 --- a/+bids/layout.m +++ b/+bids/layout.m @@ -191,7 +191,10 @@ for iFile = 1:size(file_list, 1) - [subject, parsing] = bids.internal.append_to_layout(file_list{iFile}, subject, modality, schema); + [subject, parsing] = bids.internal.append_to_layout(file_list{iFile}, ... + subject, ... + modality, ... + schema); if ~isempty(parsing) @@ -209,7 +212,8 @@ ['_asl' subject.perf(end).ext], ... '_aslcontext.tsv'); subject.(modality)(end).dependencies.context = manage_tsv( ... - struct('content', [], 'meta', []), ... + struct('content', [], ... + 'meta', []), ... pth, ... aslcontext_file); From 51d30231cb17be627fd3c5fe46e5e2ccbba47bc5 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Fri, 16 Apr 2021 21:10:08 +0200 Subject: [PATCH 22/36] rename derivate function --- +bids/{derivate.m => copy_to_derivative.m} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename +bids/{derivate.m => copy_to_derivative.m} (96%) diff --git a/+bids/derivate.m b/+bids/copy_to_derivative.m similarity index 96% rename from +bids/derivate.m rename to +bids/copy_to_derivative.m index 05f8110d..5148dd0b 100644 --- a/+bids/derivate.m +++ b/+bids/copy_to_derivative.m @@ -1,11 +1,11 @@ -function derivatives = derivate(BIDS, out_path, name, varargin) +function derivatives = copy_to_derivative(BIDS, out_path, name, varargin) % % Copy selected data from BIDS layout to given derivatives folder, % returning layout of new derivatives folder % % USAGE:: % - % derivatives = derivate(BIDS, out_path, ...) + % derivatives = copy_to_derivative(BIDS, out_path, ...) % % :param BIDS: BIDS directory name or BIDS structure (from bids.layout) % :type BIDS: (strcuture or string) From 618f2d4c17392073cace9ecec3672268b0eaee9f Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 17 Apr 2021 08:39:09 +0200 Subject: [PATCH 23/36] move files --- .gitignore | 3 --- tests/.gitignore | 6 +++++ ...ce-individual_den-native_midthickness.json | 0 ...ndividual_den-native_midthickness.surf.gii | 0 ...space-individual_den-native_thickness.json | 0 ...-individual_den-native_thickness.shape.gii | 0 ...ndividual_den-native_thickness.dscalar.nii | 0 ...space-individual_den-native_thickness.json | 0 tests/data/synthetic/CHANGES | 4 +++ tests/data/synthetic/README | 26 +++++++++++++++++++ tests/data/{MoAEpilot => synthetic}/T1w.json | 0 tests/data/synthetic/dataset_description.json | 9 +++++++ .../sub-01/anat/sub-01_T1w.json | 0 .../sub-01/anat/sub-01_T1w.nii.gz | 0 .../func/sub-01_task-auditory_bold.json | 0 .../func/sub-01_task-auditory_bold.nii.gz | 0 tests/data/synthetic/task-auditory_bold.json | 10 +++++++ tests/test_get_metadata.m | 2 +- tests/test_get_metadata_suffixes.m | 2 +- .../test_return_modality_regular_expression.m | 2 +- 20 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 tests/.gitignore rename tests/data/{SurfaceData => surface_data}/sub-06_hemi-R_space-individual_den-native_midthickness.json (100%) rename tests/data/{SurfaceData => surface_data}/sub-06_hemi-R_space-individual_den-native_midthickness.surf.gii (100%) rename tests/data/{SurfaceData => surface_data}/sub-06_hemi-R_space-individual_den-native_thickness.json (100%) rename tests/data/{SurfaceData => surface_data}/sub-06_hemi-R_space-individual_den-native_thickness.shape.gii (100%) rename tests/data/{SurfaceData => surface_data}/sub-06_space-individual_den-native_thickness.dscalar.nii (100%) rename tests/data/{SurfaceData => surface_data}/sub-06_space-individual_den-native_thickness.json (100%) create mode 100644 tests/data/synthetic/CHANGES create mode 100644 tests/data/synthetic/README rename tests/data/{MoAEpilot => synthetic}/T1w.json (100%) create mode 100644 tests/data/synthetic/dataset_description.json rename tests/data/{MoAEpilot => synthetic}/sub-01/anat/sub-01_T1w.json (100%) rename tests/data/{MoAEpilot => synthetic}/sub-01/anat/sub-01_T1w.nii.gz (100%) rename tests/data/{MoAEpilot => synthetic}/sub-01/func/sub-01_task-auditory_bold.json (100%) rename tests/data/{MoAEpilot => synthetic}/sub-01/func/sub-01_task-auditory_bold.nii.gz (100%) create mode 100644 tests/data/synthetic/task-auditory_bold.json diff --git a/.gitignore b/.gitignore index efb87036..64d4b1f1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,6 @@ *.*~ *.swp -/tests/bids-examples -/tests/*.tsv - # ignore virtual env env/* diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 00000000..4c46577d --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1,6 @@ + +*.tsv + +bids-examples + +data/MoAEpilot diff --git a/tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_midthickness.json b/tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_midthickness.json similarity index 100% rename from tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_midthickness.json rename to tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_midthickness.json diff --git a/tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_midthickness.surf.gii b/tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_midthickness.surf.gii similarity index 100% rename from tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_midthickness.surf.gii rename to tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_midthickness.surf.gii diff --git a/tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_thickness.json b/tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_thickness.json similarity index 100% rename from tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_thickness.json rename to tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_thickness.json diff --git a/tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_thickness.shape.gii b/tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_thickness.shape.gii similarity index 100% rename from tests/data/SurfaceData/sub-06_hemi-R_space-individual_den-native_thickness.shape.gii rename to tests/data/surface_data/sub-06_hemi-R_space-individual_den-native_thickness.shape.gii diff --git a/tests/data/SurfaceData/sub-06_space-individual_den-native_thickness.dscalar.nii b/tests/data/surface_data/sub-06_space-individual_den-native_thickness.dscalar.nii similarity index 100% rename from tests/data/SurfaceData/sub-06_space-individual_den-native_thickness.dscalar.nii rename to tests/data/surface_data/sub-06_space-individual_den-native_thickness.dscalar.nii diff --git a/tests/data/SurfaceData/sub-06_space-individual_den-native_thickness.json b/tests/data/surface_data/sub-06_space-individual_den-native_thickness.json similarity index 100% rename from tests/data/SurfaceData/sub-06_space-individual_den-native_thickness.json rename to tests/data/surface_data/sub-06_space-individual_den-native_thickness.json diff --git a/tests/data/synthetic/CHANGES b/tests/data/synthetic/CHANGES new file mode 100644 index 00000000..cc28676a --- /dev/null +++ b/tests/data/synthetic/CHANGES @@ -0,0 +1,4 @@ +1.0.1 2019-03-20 + - BIDS version. +1.0.0 1999-05-13 + - Initial release. diff --git a/tests/data/synthetic/README b/tests/data/synthetic/README new file mode 100644 index 00000000..959ca6ba --- /dev/null +++ b/tests/data/synthetic/README @@ -0,0 +1,26 @@ + ___ ____ __ __ +/ __)( _ \( \/ ) Statistical Parametric Mapping +\__ \ )___/ ) ( Wellcome Centre for Human Neuroimaging +(___/(__) (_/\/\_) https://www.fil.ion.ucl.ac.uk/spm/ + + MoAEpilot example epoch (block) fMRI dataset +________________________________________________________________________ + +This experiment was conducted by Geraint Rees under the direction of +Karl Friston and the FIL methods group. The purpose was to explore new +equipment and techniques. As such it has not been formally written up, +and is freely available for personal education and evaluation purposes. + +These whole brain BOLD/EPI images were acquired on a modified 2T +Siemens MAGNETOM Vision system. Each acquisition consisted of 64 +contiguous slices (64x64x64 3mm x 3mm x 3mm voxels). Acquisition took +6.05s, with the scan to scan repeat time (RT) set arbitrarily to 7s. + +96 acquisitions were made (RT=7s), in blocks of 6, giving 16 42s +blocks. The condition for successive blocks alternated between rest and +auditory stimulation, starting with rest. Auditory stimulation was +bi-syllabic words presented binaurally at a rate of 60 per minute. Due +to T1 effects it is advisable to discard the first few scans (there were +no "dummy" lead-in scans). + +A structural image was also acquired. diff --git a/tests/data/MoAEpilot/T1w.json b/tests/data/synthetic/T1w.json similarity index 100% rename from tests/data/MoAEpilot/T1w.json rename to tests/data/synthetic/T1w.json diff --git a/tests/data/synthetic/dataset_description.json b/tests/data/synthetic/dataset_description.json new file mode 100644 index 00000000..7948d385 --- /dev/null +++ b/tests/data/synthetic/dataset_description.json @@ -0,0 +1,9 @@ +{ + "BIDSVersion": "1.2.0", + "Name": "Mother of All Experiments", + "Authors": [ + "Geraint Rees", + "Karl Friston" + ], + "ReferencesAndLinks": ["https://www.fil.ion.ucl.ac.uk/spm/data/auditory/"] +} \ No newline at end of file diff --git a/tests/data/MoAEpilot/sub-01/anat/sub-01_T1w.json b/tests/data/synthetic/sub-01/anat/sub-01_T1w.json similarity index 100% rename from tests/data/MoAEpilot/sub-01/anat/sub-01_T1w.json rename to tests/data/synthetic/sub-01/anat/sub-01_T1w.json diff --git a/tests/data/MoAEpilot/sub-01/anat/sub-01_T1w.nii.gz b/tests/data/synthetic/sub-01/anat/sub-01_T1w.nii.gz similarity index 100% rename from tests/data/MoAEpilot/sub-01/anat/sub-01_T1w.nii.gz rename to tests/data/synthetic/sub-01/anat/sub-01_T1w.nii.gz diff --git a/tests/data/MoAEpilot/sub-01/func/sub-01_task-auditory_bold.json b/tests/data/synthetic/sub-01/func/sub-01_task-auditory_bold.json similarity index 100% rename from tests/data/MoAEpilot/sub-01/func/sub-01_task-auditory_bold.json rename to tests/data/synthetic/sub-01/func/sub-01_task-auditory_bold.json diff --git a/tests/data/MoAEpilot/sub-01/func/sub-01_task-auditory_bold.nii.gz b/tests/data/synthetic/sub-01/func/sub-01_task-auditory_bold.nii.gz similarity index 100% rename from tests/data/MoAEpilot/sub-01/func/sub-01_task-auditory_bold.nii.gz rename to tests/data/synthetic/sub-01/func/sub-01_task-auditory_bold.nii.gz diff --git a/tests/data/synthetic/task-auditory_bold.json b/tests/data/synthetic/task-auditory_bold.json new file mode 100644 index 00000000..68017a5b --- /dev/null +++ b/tests/data/synthetic/task-auditory_bold.json @@ -0,0 +1,10 @@ +{ + "RepetitionTime": 7, + "NumberOfVolumesDiscardedByScanner": 0, + "NumberOfVolumesDiscardedByUser": 12, + "TaskName": "Auditory", + "TaskDescription": "The condition for successive blocks alternated between rest and auditory stimulation, starting with rest. Auditory stimulation was with bi-syllabic words presented binaurally at a rate of 60 per minute.", + "Manufacturer": "Siemens", + "ManufacturersModelName": "MAGNETOM Vision", + "MagneticFieldStrength": 2 +} \ No newline at end of file diff --git a/tests/test_get_metadata.m b/tests/test_get_metadata.m index 50b31ec7..34260e3d 100644 --- a/tests/test_get_metadata.m +++ b/tests/test_get_metadata.m @@ -42,7 +42,7 @@ function test_get_metadata_basic() % also tests inheritance principle: metadata are passed on to lower levels % unless they are overriden by metadate already present at lower levels - pth = fullfile(fileparts(mfilename('fullpath')), 'data', 'MoAEpilot'); + pth = fullfile(fileparts(mfilename('fullpath')), 'data', 'synthetic'); % define the expected output from bids query metadata func.RepetitionTime = 7; diff --git a/tests/test_get_metadata_suffixes.m b/tests/test_get_metadata_suffixes.m index e43f36c3..90448db8 100644 --- a/tests/test_get_metadata_suffixes.m +++ b/tests/test_get_metadata_suffixes.m @@ -9,7 +9,7 @@ function test_get_metadata_suffixes_basic() % ensures that "similar" suffixes are distinguished - data_dir = fullfile(fileparts(mfilename('fullpath')), 'data', 'SurfaceData'); + data_dir = fullfile(fileparts(mfilename('fullpath')), 'data', 'surface_data'); file = fullfile(data_dir, 'sub-06_hemi-R_space-individual_den-native_thickness.shape.gii'); side_car = fullfile(data_dir, 'sub-06_hemi-R_space-individual_den-native_thickness.json'); diff --git a/tests/test_return_modality_regular_expression.m b/tests/test_return_modality_regular_expression.m index b3984f5b..6b8b8edd 100644 --- a/tests/test_return_modality_regular_expression.m +++ b/tests/test_return_modality_regular_expression.m @@ -20,7 +20,7 @@ assertEqual(regular_expression, expected_expression); - data_dir = fullfile(fileparts(mfilename('fullpath')), 'data', 'MoAEpilot', 'sub-01', 'anat'); + data_dir = fullfile(fileparts(mfilename('fullpath')), 'data', 'synthetic', 'sub-01', 'anat'); subject_name = 'sub-01'; file = bids.internal.file_utils('List', data_dir, sprintf(expected_expression, subject_name)); From e1d1c8a2c3f0ea2c5331f3011bac80ce476e2fe6 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 17 Apr 2021 08:41:08 +0200 Subject: [PATCH 24/36] reimplement getting metadata from files in root folder --- +bids/+internal/get_meta_list.m | 25 +++++++++++-------- tests/test_get_metadata.m | 35 ++++++++++++--------------- tests/test_get_metadata_suffixes.m | 39 ++++++++++++++---------------- 3 files changed, 49 insertions(+), 50 deletions(-) diff --git a/+bids/+internal/get_meta_list.m b/+bids/+internal/get_meta_list.m index c5948cf4..f4bc581e 100644 --- a/+bids/+internal/get_meta_list.m +++ b/+bids/+internal/get_meta_list.m @@ -27,18 +27,23 @@ p = bids.internal.parse_filename(filename); metalist = {}; - N = 3; - - % -There is a session level in the hierarchy - if isfield(p.entities, 'ses') && ~isempty(p.entities.ses) - N = N + 1; + % Default assumes we are dealing with a file in the root directory + % like "participants.tsv" + % If the file has underscore separated entities ("sub-01_T1w.nii") + % then we look through the hierarchy for potential metadata file associated + % with queried file. + N = 1; + if isfield(p, 'entities') + N = 3; + % -There is a session level in the hierarchy + if isfield(p.entities, 'ses') && ~isempty(p.entities.ses) + N = N + 1; + end end - % -Loop from the directory where the file of interest is back to the - % top level of the BIDS hierarchy for n = 1:N - % -List the potential metadata files associated with this file suffix type + % List the potential metadata files associated with this file suffix type % Default is to assume it is a JSON file metafile = bids.internal.file_utils('FPList', pth, sprintf(pattern, p.suffix)); @@ -48,7 +53,7 @@ metafile = cellstr(metafile); end - % -For all those files we find which one is potentially associated with + % For all those files we find which one is potentially associated with % the file of interest for i = 1:numel(metafile) @@ -81,7 +86,7 @@ end - % -Go up to the parent folder + % Go up to the parent folder pth = fullfile(pth, '..'); end end diff --git a/tests/test_get_metadata.m b/tests/test_get_metadata.m index 34260e3d..186bb9b3 100644 --- a/tests/test_get_metadata.m +++ b/tests/test_get_metadata.m @@ -6,25 +6,6 @@ initTestSuite; end -function test_get_metadata_participants() - % test files with no underscore in name. - - pth_bids_example = get_test_data_dir(); - - file = fullfile(pth_bids_example, 'ds001', 'participants.tsv'); - side_car = fullfile(pth_bids_example, 'ds001', 'participants.json'); - - % SILENCING TEST - % bids.internal.get_metadata now only takes .jsons as input - % bids.internal.get_meta_list is now in charge of building the list of - % metadata file list - - % metadata = bids.internal.get_metadata(file); - % expected_metadata = bids.util.jsondecode(side_car); - % assertEqual(metadata, expected_metadata); - -end - function test_get_metadata_basic() % Test metadata and the inheritance principle % __________________________________________________________________________ @@ -83,3 +64,19 @@ function test_get_metadata_internal() bids.internal.get_metadata(BIDS(1).subjects(2).anat(1).metafile); end + +function test_get_metadata_participants() + % test files with no underscore in name. + + pth_bids_example = get_test_data_dir(); + + file = fullfile(pth_bids_example, 'ds001', 'participants.tsv'); + side_car = fullfile(pth_bids_example, 'ds001', 'participants.json'); + + metalist = bids.internal.get_meta_list(file); + metadata = bids.internal.get_metadata(metalist); + + expected_metadata = bids.util.jsondecode(side_car); + assertEqual(metadata, expected_metadata); + +end diff --git a/tests/test_get_metadata_suffixes.m b/tests/test_get_metadata_suffixes.m index 90448db8..7baf2221 100644 --- a/tests/test_get_metadata_suffixes.m +++ b/tests/test_get_metadata_suffixes.m @@ -14,34 +14,31 @@ function test_get_metadata_suffixes_basic() file = fullfile(data_dir, 'sub-06_hemi-R_space-individual_den-native_thickness.shape.gii'); side_car = fullfile(data_dir, 'sub-06_hemi-R_space-individual_den-native_thickness.json'); - % SILENCING TEST - % bids.internal.get_metadata now only takes .jsons as input - % bids.internal.get_meta_list is now in charge of building the list of - % metadata file list - % metadata = bids.internal.get_metadata(file); - % expected_metadata = bids.util.jsondecode(side_car); - % assertEqual(metadata, expected_metadata); + metalist = bids.internal.get_meta_list(file); + metadata = bids.internal.get_metadata(metalist); + + expected_metadata = bids.util.jsondecode(side_car); + + assertEqual(metadata, expected_metadata); file = fullfile(data_dir, 'sub-06_hemi-R_space-individual_den-native_midthickness.surf.gii'); side_car = fullfile(data_dir, 'sub-06_hemi-R_space-individual_den-native_midthickness.json'); - % SILENCING TEST - % bids.internal.get_metadata now only takes .jsons as input - % bids.internal.get_meta_list is now in charge of building the list of - % metadata file list - % metadata = bids.internal.get_metadata(file); - % expected_metadata = bids.util.jsondecode(side_car); - % assertEqual(metadata, expected_metadata); + metalist = bids.internal.get_meta_list(file); + metadata = bids.internal.get_metadata(metalist); + + expected_metadata = bids.util.jsondecode(side_car); + + assertEqual(metadata, expected_metadata); file = fullfile(data_dir, 'sub-06_space-individual_den-native_thickness.dscalar.nii'); side_car = fullfile(data_dir, 'sub-06_space-individual_den-native_thickness.json'); - % SILENCING TEST - % bids.internal.get_metadata now only takes .jsons as input - % bids.internal.get_meta_list is now in charge of building the list of - % metadata file list - % metadata = bids.internal.get_metadata(file); - % expected_metadata = bids.util.jsondecode(side_car); - % assertEqual(metadata, expected_metadata); + metalist = bids.internal.get_meta_list(file); + metadata = bids.internal.get_metadata(metalist); + + expected_metadata = bids.util.jsondecode(side_car); + + assertEqual(metadata, expected_metadata); end From 1baa5d9ca80f3dfca39c162555e932e5b53e17c9 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 17 Apr 2021 08:44:14 +0200 Subject: [PATCH 25/36] add test to copy_to_derivative --- +bids/copy_to_derivative.m | 54 ++++++++++++++++++--------------- .github/workflows/run_tests.yml | 3 +- tests/download_moae_ds.m | 34 +++++++++++++++++++++ tests/test_copy_to_derivative.m | 27 +++++++++++++++++ 4 files changed, 93 insertions(+), 25 deletions(-) create mode 100644 tests/download_moae_ds.m create mode 100644 tests/test_copy_to_derivative.m diff --git a/+bids/copy_to_derivative.m b/+bids/copy_to_derivative.m index 5148dd0b..ef8c2f6a 100644 --- a/+bids/copy_to_derivative.m +++ b/+bids/copy_to_derivative.m @@ -31,7 +31,7 @@ BIDS = bids.layout(BIDS); if ~exist(out_path, 'dir') - error(['Output path ' out_path ' not found']); + mkdir(out_path); end derivatives = []; @@ -39,7 +39,7 @@ subjects_list = bids.query(BIDS, 'subjects', varargin{:}); if isempty(data_list) - warning(['No data found for this query']); + warning('No data found for this query'); return else fprintf('Found %d files in %d subjects\n', length(data_list), length(subjects_list)); @@ -50,27 +50,25 @@ mkdir(pth_BIDSderiv); end - % creating description + % creating / loading description descr_file = fullfile(pth_BIDSderiv, 'dataset_description.json'); - pipeline.Name = mfilename; - % pipeline.Verion = ? - pipeline.Container = varargin; - - % loading dataset description if exist(descr_file, 'file') description = bids.util.jsondecode(descr_file); else description = BIDS.description; end - % updating GeneratedBy + % Create / update GeneratedBy + pipeline.Name = mfilename; + pipeline.Version = ''; + pipeline.Container = varargin; if isfield(description, 'GeneratedBy') description.GeneratedBy = [description.GeneratedBy pipeline]; else - description.GeneratedBy = [pipeline]; + description.GeneratedBy = pipeline; end - bids.util.jsonencode(descr_file, description, 'Indent', ' '); + bids.util.jsonencode(descr_file, description, struct('Indent', ' ')); % extracting participants.tsv file? @@ -82,45 +80,53 @@ end function status = copy_file(BIDS, derivatives_folder, data_file) - status = 1; + + status = true; + info = bids.internal.return_file_info(BIDS, data_file); file = BIDS.subjects(info.sub_idx).(info.modality)(info.file_idx); - basename = file.filename(1:end - length(file.ext)); + out_dir = fullfile(derivatives_folder, ... BIDS.subjects(info.sub_idx).name, ... BIDS.subjects(info.sub_idx).session, ... info.modality); - out_path = fullfile(out_dir, basename); - meta_file = [out_path '.json']; - % ignore already existing files; avoid circular references - if exist(meta_file) + meta_file = fullfile(out_dir, [bids.internal.file_utils(file.filename, 'basename') '.json']); + + %% ignore already existing files + % avoid circular references + if exist(meta_file, 'file') return + else + file.meta = bids.internal.get_metadata(file.metafile); end + if ~exist(out_dir, 'dir') mkdir(out_dir); end - % copy data file + + %% copy data file if endsWith(file.ext, '.gz') + % might be an issue with octave that removes original file gunzip(data_file, out_dir); else - [status, message, messageId] = copyfile(data_file, [out_path file.ext]); + [status, message, messageId] = copyfile(data_file, fullfile(out_dir, file.filename)); end if ~status warning([messageId ': ' message]); return end - % export metadata + + %% export metadata if ~strcmpi(file.ext, '.json') % skip if data file is json bids.util.jsonencode(meta_file, file.meta); end - % checking that json is created - if ~exist(meta_file) - error(['Failed to create sidecar json file: ' meta_file]); + if ~exist(meta_file, 'file') + error('Failed to create sidecar json file: %s', meta_file); end - % trating depandencies + %% dealing with dependencies if ~isempty(file.dependencies) dependencies = fieldnames(file.dependencies); for dep = 1:numel(dependencies) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 88e50fee..073ee5d2 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -42,10 +42,11 @@ jobs: mkoctfile --mex jsonread.c jsmn.c -DJSMN_PARENT_LINKS cd .. - - name: Install bids example + - name: Install bids-example and data run: | cd tests git clone git://github.com/bids-standard/bids-examples.git --depth 1 + octave $OCTFLAGS --eval "download_moae_ds" cd .. - name: MOxUnit Action diff --git a/tests/download_moae_ds.m b/tests/download_moae_ds.m new file mode 100644 index 00000000..5ab3d957 --- /dev/null +++ b/tests/download_moae_ds.m @@ -0,0 +1,34 @@ +function output_dir = download_moae_ds(downloadData) + % + % Copyright (C) 2021--, BIDS-MATLAB developers + + if nargin < 1 + downloadData = true(); + end + + output_dir = fullfile(fileparts(mfilename('fullpath')), 'data'); + + if downloadData + + % URL of the data set to download + URL = 'http://www.fil.ion.ucl.ac.uk/spm/download/data/MoAEpilot/MoAEpilot.bids.zip'; + + % clean previous runs + if exist(fullfile(output_dir, 'MoAEpilot'), 'dir') + rmdir(fullfile(output_dir, 'MoAEpilot'), 's'); + end + + %% Get data + fprintf('%-10s:', 'Downloading dataset...'); + urlwrite(URL, 'MoAEpilot.zip'); + fprintf(1, ' Done\n\n'); + + fprintf('%-10s:', 'Unzipping dataset...'); + unzip('MoAEpilot.zip'); + delete('MoAEpilot.zip'); + movefile('MoAEpilot', fullfile(output_dir)); + fprintf(1, ' Done\n\n'); + + end + +end diff --git a/tests/test_copy_to_derivative.m b/tests/test_copy_to_derivative.m new file mode 100644 index 00000000..3704add5 --- /dev/null +++ b/tests/test_copy_to_derivative.m @@ -0,0 +1,27 @@ +function test_suite = test_copy_to_derivative %#ok<*STOUT> + try % assignment of 'localfunctions' is necessary in Matlab >= 2016 + test_functions = localfunctions(); %#ok<*NASGU> + catch % no problem; early Matlab versions can use initTestSuite fine + end + initTestSuite; +end + +function test_copy_to_derivative_basic() + + input_dir = download_moae_ds(true()); + out_path = fullfile(input_dir, 'MoAEpilot', 'derivatives'); + + BIDS = fullfile(input_dir, 'MoAEpilot'); + + pipeline_name = 'bids-matlab'; + + derivatives = bids.copy_to_derivative(BIDS, out_path, pipeline_name); + +end + +% +% function test_copy_to_derivative_fmriprep() +% +% +% +% end From bcb7f1f3523a0c07c405ddeb20d6036fdadd342a Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 17 Apr 2021 09:57:47 +0200 Subject: [PATCH 26/36] add instructions to get test data --- tests/.gitignore | 4 +++- tests/README.md | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/.gitignore b/tests/.gitignore index 4c46577d..f303df46 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -3,4 +3,6 @@ bids-examples -data/MoAEpilot +data/MoAEpilot/* +data/ds*/* +data/derivatives*/* diff --git a/tests/README.md b/tests/README.md index 02844abc..ddd02eaa 100644 --- a/tests/README.md +++ b/tests/README.md @@ -52,6 +52,19 @@ cd tests git clone git://github.com/bids-standard/bids-examples.git --depth 1 ``` +#### Datasets with content + +The function `download_moae_ds.m` will download a lightweight dataset from the +SPM website. + +To get more complex data sets, to test things you can use datalad. + +```bash +cd tests/data +datalad clone https://github.com/OpenNeuroDatasets/ds000001.git +datalad get ds000001/sub-01/ +``` + ## Add helper functions to the path There are a some help functions you need to add to the Matlab / Octave path to From a01234bafd99ba97db9fc9a724cb2012b5e621d9 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 17 Apr 2021 10:44:34 +0200 Subject: [PATCH 27/36] filters for query can be passed as a n X 2 cell --- +bids/query.m | 31 +++++++++++++++++++-- tests/test_bids_query.m | 61 +++++++++++++++++++++++++++-------------- 2 files changed, 69 insertions(+), 23 deletions(-) diff --git a/+bids/query.m b/+bids/query.m index 0e521451..c73064f7 100644 --- a/+bids/query.m +++ b/+bids/query.m @@ -19,8 +19,27 @@ % - 'modalities' % :type query: string % + % Queries can "filtered" by passing more arguments key-value pairs as a list of + % strings or as a cell or a structure + % + % Example 1:: + % + % data = bids.query(BIDS, 'data', ... + % 'sub', '01', ... + % 'task', 'stopsignalwithpseudowordnaming', ... + % 'extension', '.nii.gz', ... + % 'suffix', 'bold'); + % + % + % Example 2:: + % + % filters = struct('sub', '01', ... + % 'task', 'stopsignalwithpseudowordnaming', ... + % 'extension', '.nii.gz', ... + % 'suffix', 'bold'); + % + % data = bids.query(BIDS, 'data', filters); % - % __________________________________________________________________________ % % BIDS (Brain Imaging Data Structure): https://bids.neuroimaging.io/ @@ -100,8 +119,14 @@ function options = parse_query(options) - if numel(options) == 1 && isstruct(options{1}) - options = [fieldnames(options{1}), struct2cell(options{1})]; + if numel(options) == 1 + + if isstruct(options{1}) + options = [fieldnames(options{1}), struct2cell(options{1})]; + + elseif iscell(options{1}) + options = options{1}; + end else if mod(numel(options), 2) diff --git a/tests/test_bids_query.m b/tests/test_bids_query.m index f46dbff5..912b197d 100644 --- a/tests/test_bids_query.m +++ b/tests/test_bids_query.m @@ -10,6 +10,47 @@ end +function test_query_basic() + + pth_bids_example = get_test_data_dir(); + + BIDS = bids.layout(fullfile(pth_bids_example, 'ds007')); + + tasks = { ... + 'stopsignalwithletternaming', ... + 'stopsignalwithmanualresponse', ... + 'stopsignalwithpseudowordnaming'}; + assertEqual(bids.query(BIDS, 'tasks'), tasks); + + assert(isempty(bids.query(BIDS, 'runs', 'suffix', 'T1w'))); + + runs = {'01', '02'}; + assertEqual(bids.query(BIDS, 'runs'), runs); + assertEqual(bids.query(BIDS, 'runs', 'suffix', 'bold'), runs); + + % make sure that query can work with filter + filters = {'sub', {'01', '03'}; ... + 'task', {'stopsignalwithletternaming', ... + 'stopsignalwithmanualresponse'}; ... + 'run', '02'; ... + 'suffix', 'bold'}; + + files_cell_filter = bids.query(BIDS, 'data', filters); + assertEqual(size(files_cell_filter, 1), 4); + + filters = struct('run', '02', ... + 'suffix', 'bold'); + filters.sub = {'01', '03'}; + filters.task = {'stopsignalwithletternaming', ... + 'stopsignalwithmanualresponse'}; + + files_struct_filter = bids.query(BIDS, 'data', filters); + assertEqual(size(files_struct_filter, 1), 4); + + assertEqual(files_cell_filter, files_struct_filter); + +end + function test_query_extension() pth_bids_example = get_test_data_dir(); @@ -108,26 +149,6 @@ function test_query_modalities() end -function test_query_basic() - - pth_bids_example = get_test_data_dir(); - - BIDS = bids.layout(fullfile(pth_bids_example, 'ds007')); - - tasks = { ... - 'stopsignalwithletternaming', ... - 'stopsignalwithmanualresponse', ... - 'stopsignalwithpseudowordnaming'}; - assertEqual(bids.query(BIDS, 'tasks'), tasks); - - assert(isempty(bids.query(BIDS, 'runs', 'suffix', 'T1w'))); - - runs = {'01', '02'}; - assertEqual(bids.query(BIDS, 'runs'), runs); - assertEqual(bids.query(BIDS, 'runs', 'suffix', 'bold'), runs); - -end - function test_query_subjects() pth_bids_example = get_test_data_dir(); From 4c9d7b9427e6d24c235a588837fe74b47e9b9441 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 17 Apr 2021 10:48:06 +0200 Subject: [PATCH 28/36] add test for copy_to_derivatives on real data --- +bids/copy_to_derivative.m | 118 ++++++++++++++++++++--------- .github/workflows/run_tests.yml | 6 +- tests/download_moae_ds.m | 3 + tests/test_copy_to_derivative.m | 27 +++++-- tests/test_get_metadata_suffixes.m | 12 +-- 5 files changed, 117 insertions(+), 49 deletions(-) diff --git a/+bids/copy_to_derivative.m b/+bids/copy_to_derivative.m index ef8c2f6a..11502832 100644 --- a/+bids/copy_to_derivative.m +++ b/+bids/copy_to_derivative.m @@ -1,18 +1,20 @@ -function derivatives = copy_to_derivative(BIDS, out_path, name, varargin) +function derivatives = copy_to_derivative(BIDS, out_path, pipeline_name, query) % % Copy selected data from BIDS layout to given derivatives folder, % returning layout of new derivatives folder % % USAGE:: % - % derivatives = copy_to_derivative(BIDS, out_path, ...) + % derivatives = copy_to_derivative(BIDS, out_path, query) % - % :param BIDS: BIDS directory name or BIDS structure (from bids.layout) - % :type BIDS: (strcuture or string) - % :param out_path: path to directory containing the derivatives - % :type out_path: string - % :param name: name of pipeline to use - % :type name: string + % :param BIDS: BIDS directory name or BIDS structure (from bids.layout) + % :type BIDS: structure or string + % :param out_path: path to directory containing the derivatives + % :type out_path: string + % :param pipeline_name: name of pipeline to use + % :type pipeline_name: string + % :param pipeline_name: list of filters to choose what files to copy (see bids.query) + % :type pipeline_name: structure or cell % % % __________________________________________________________________________ @@ -35,8 +37,8 @@ end derivatives = []; - data_list = bids.query(BIDS, 'data', varargin{:}); - subjects_list = bids.query(BIDS, 'subjects', varargin{:}); + data_list = bids.query(BIDS, 'data', query); + subjects_list = bids.query(BIDS, 'subjects', query); if isempty(data_list) warning('No data found for this query'); @@ -45,13 +47,13 @@ fprintf('Found %d files in %d subjects\n', length(data_list), length(subjects_list)); end - pth_BIDSderiv = fullfile(out_path, name); - if ~exist(pth_BIDSderiv, 'dir') - mkdir(pth_BIDSderiv); + derivatives_folder = fullfile(out_path, pipeline_name); + if ~exist(derivatives_folder, 'dir') + mkdir(derivatives_folder); end % creating / loading description - descr_file = fullfile(pth_BIDSderiv, 'dataset_description.json'); + descr_file = fullfile(derivatives_folder, 'dataset_description.json'); if exist(descr_file, 'file') description = bids.util.jsondecode(descr_file); else @@ -61,7 +63,7 @@ % Create / update GeneratedBy pipeline.Name = mfilename; pipeline.Version = ''; - pipeline.Container = varargin; + pipeline.Container = ''; if isfield(description, 'GeneratedBy') description.GeneratedBy = [description.GeneratedBy pipeline]; else @@ -74,14 +76,12 @@ % looping over selected files for iFile = 1:numel(data_list) - copy_file(BIDS, pth_BIDSderiv, data_list{iFile}); + copy_file(BIDS, derivatives_folder, data_list{iFile}); end end -function status = copy_file(BIDS, derivatives_folder, data_file) - - status = true; +function copy_file(BIDS, derivatives_folder, data_file) info = bids.internal.return_file_info(BIDS, data_file); file = BIDS.subjects(info.sub_idx).(info.modality)(info.file_idx); @@ -91,11 +91,12 @@ BIDS.subjects(info.sub_idx).session, ... info.modality); - meta_file = fullfile(out_dir, [bids.internal.file_utils(file.filename, 'basename') '.json']); + output_metadata_file = fullfile(out_dir, ... + strrep(file.filename, file.ext, '.json')); %% ignore already existing files % avoid circular references - if exist(meta_file, 'file') + if exist(output_metadata_file, 'file') return else file.meta = bids.internal.get_metadata(file.metafile); @@ -106,32 +107,29 @@ end %% copy data file - if endsWith(file.ext, '.gz') - % might be an issue with octave that removes original file - gunzip(data_file, out_dir); - else - [status, message, messageId] = copyfile(data_file, fullfile(out_dir, file.filename)); - end - if ~status - warning([messageId ': ' message]); - return - end + % we follow any eventual symlink + % and then unzip the data if necessary + copy_with_symlink(data_file, fullfile(out_dir, file.filename)); + unzip_data(file, out_dir); %% export metadata + % All the metadata of each file is read through the whole hierarchy + % and dumped into one side-car json file for each file copied + % In practice this "unravels" the inheritance principle if ~strcmpi(file.ext, '.json') % skip if data file is json - bids.util.jsonencode(meta_file, file.meta); + bids.util.jsonencode(output_metadata_file, file.meta); end % checking that json is created - if ~exist(meta_file, 'file') - error('Failed to create sidecar json file: %s', meta_file); + if ~exist(output_metadata_file, 'file') + error('Failed to create sidecar json file: %s', output_metadata_file); end %% dealing with dependencies if ~isempty(file.dependencies) dependencies = fieldnames(file.dependencies); for dep = 1:numel(dependencies) - for idep = 1:numel(file.dependencies.(dependencies{dep})) - dep_file = file.dependencies.(dependencies{dep}){idep}; + for ifile = 1:numel(file.dependencies.(dependencies{dep})) + dep_file = file.dependencies.(dependencies{dep}){ifile}; if exist(dep_file, 'file') copy_file(BIDS, derivatives_folder, dep_file); else @@ -142,3 +140,51 @@ end end + +function copy_with_symlink(src, target) + % + % Follows symbolic link to copy data: + % Might be necessary for datasets curated with datalad + % + + command = 'cp -R -L -f'; + + try + status = system( ... + sprintf('%s %s %s', ... + command, ... + src, ... + target)); + + if status > 0 + message = [ ... + 'Copying data with system command failed: ' ... + 'Are you running Windows?\n', ... + 'Will use matlab/octave copyfile command instead.\n', ... + 'May be an issue if your data set contains symbolic links' ... + '(e.g. if you use datalad or git-annex.)']; + error(message); + end + + catch + + fprintf(1, 'Using octave/matlab to copy files.'); + [status, message, messageId] = copyfile(src, target); + if ~status + warning([messageId ': ' message]); + return + end + + end + +end + +function unzip_data(file, out_dir) + % to ensure a consistent behavior with matlab and octave + if bids.internal.ends_with(file.ext, '.gz') + gunzip(fullfile(out_dir, file.filename)); + if exist(fullfile(out_dir, file.filename), 'file') + delete(fullfile(out_dir, file.filename)); + end + end +end diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 073ee5d2..1eb61343 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -24,6 +24,7 @@ jobs: sudo apt-get -y -qq update sudo apt-get -y install octave liboctave-dev sudo apt-get -y install nodejs npm + sudo apt-get -y install datalad - name: Clone bids-matlab uses: actions/checkout@v2 @@ -47,7 +48,10 @@ jobs: cd tests git clone git://github.com/bids-standard/bids-examples.git --depth 1 octave $OCTFLAGS --eval "download_moae_ds" - cd .. + cd data + datalad clone https://github.com/OpenNeuroDatasets/ds000001.git + datalad get ds000001/sub-01/func + cd ../.. - name: MOxUnit Action uses: joergbrech/moxunit-action@master diff --git a/tests/download_moae_ds.m b/tests/download_moae_ds.m index 5ab3d957..6f865420 100644 --- a/tests/download_moae_ds.m +++ b/tests/download_moae_ds.m @@ -1,4 +1,7 @@ function output_dir = download_moae_ds(downloadData) + % + % Will download the lightweight "Mother of all experiment" dataset from the + % SPM website. % % Copyright (C) 2021--, BIDS-MATLAB developers diff --git a/tests/test_copy_to_derivative.m b/tests/test_copy_to_derivative.m index 3704add5..b47505f1 100644 --- a/tests/test_copy_to_derivative.m +++ b/tests/test_copy_to_derivative.m @@ -6,22 +6,37 @@ initTestSuite; end -function test_copy_to_derivative_basic() +function test_copy_to_derivative_ds000001() - input_dir = download_moae_ds(true()); - out_path = fullfile(input_dir, 'MoAEpilot', 'derivatives'); + input_dir = fullfile(pwd, 'data', 'ds000001'); + out_path = fullfile(pwd, 'data', 'derivatives'); - BIDS = fullfile(input_dir, 'MoAEpilot'); + if exist(out_path, 'dir') + rmdir(out_path, 's'); + end + + BIDS = fullfile(input_dir); + + what_to_copy = struct('sub', '01', ... + 'modality', 'func', ... + 'suffix', 'bold'); + what_to_copy.run = {'01'; '03'}; pipeline_name = 'bids-matlab'; - derivatives = bids.copy_to_derivative(BIDS, out_path, pipeline_name); + derivatives = bids.copy_to_derivative(BIDS, out_path, pipeline_name, what_to_copy); end +% function test_copy_to_derivative_MoAE() +% +% input_dir = download_moae_ds(true()); +% out_path = fullfile(input_dir, 'MoAEpilot', 'derivatives'); % -% function test_copy_to_derivative_fmriprep() +% BIDS = fullfile(input_dir, 'MoAEpilot'); % +% pipeline_name = 'bids-matlab'; % +% derivatives = bids.copy_to_derivative(BIDS, out_path, pipeline_name); % % end diff --git a/tests/test_get_metadata_suffixes.m b/tests/test_get_metadata_suffixes.m index 7baf2221..9ee503f4 100644 --- a/tests/test_get_metadata_suffixes.m +++ b/tests/test_get_metadata_suffixes.m @@ -16,9 +16,9 @@ function test_get_metadata_suffixes_basic() metalist = bids.internal.get_meta_list(file); metadata = bids.internal.get_metadata(metalist); - + expected_metadata = bids.util.jsondecode(side_car); - + assertEqual(metadata, expected_metadata); file = fullfile(data_dir, 'sub-06_hemi-R_space-individual_den-native_midthickness.surf.gii'); @@ -26,9 +26,9 @@ function test_get_metadata_suffixes_basic() metalist = bids.internal.get_meta_list(file); metadata = bids.internal.get_metadata(metalist); - + expected_metadata = bids.util.jsondecode(side_car); - + assertEqual(metadata, expected_metadata); file = fullfile(data_dir, 'sub-06_space-individual_den-native_thickness.dscalar.nii'); @@ -36,9 +36,9 @@ function test_get_metadata_suffixes_basic() metalist = bids.internal.get_meta_list(file); metadata = bids.internal.get_metadata(metalist); - + expected_metadata = bids.util.jsondecode(side_car); - + assertEqual(metadata, expected_metadata); end From 7f0d9b10a51bd531c1e4a62b8dd20e11e7973214 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 17 Apr 2021 11:41:08 +0200 Subject: [PATCH 29/36] add option to unzip and overwrite when copying to derivatives --- +bids/copy_to_derivative.m | 87 ++++++++++++++++++++++++++------- +bids/query.m | 4 ++ tests/test_copy_to_derivative.m | 52 ++++++++++++-------- 3 files changed, 103 insertions(+), 40 deletions(-) diff --git a/+bids/copy_to_derivative.m b/+bids/copy_to_derivative.m index 11502832..1cff3ca9 100644 --- a/+bids/copy_to_derivative.m +++ b/+bids/copy_to_derivative.m @@ -1,4 +1,4 @@ -function derivatives = copy_to_derivative(BIDS, out_path, pipeline_name, query) +function derivatives = copy_to_derivative(BIDS, out_path, pipeline_name, filter, unzip, force, verbose) % % Copy selected data from BIDS layout to given derivatives folder, % returning layout of new derivatives folder @@ -13,8 +13,17 @@ % :type out_path: string % :param pipeline_name: name of pipeline to use % :type pipeline_name: string - % :param pipeline_name: list of filters to choose what files to copy (see bids.query) - % :type pipeline_name: structure or cell + % :param query: list of filters to choose what files to copy (see bids.query) + % :type query: structure or cell + % :param unzip: If ``true`` (default) then all ``.gz`` files will be unzipped + % after being copied. + % :type unzip: boolean + % + % All the metadata of each file is read through the whole hierarchy + % and dumped into one side-car json file for each file copied. + % In practice this "unravels" the inheritance principle. + % The presence of this metadata file is also used to prevent the file from + % being copied again on a successive run. % % % __________________________________________________________________________ @@ -26,19 +35,35 @@ % __________________________________________________________________________ % Copyright (C) 2016-2018, Guillaume Flandin, Wellcome Centre for Human Neuroimaging - % Copyright (C) 2018--, BIDS-MATLAB developers + % Copyright (C) 2021--, BIDS-MATLAB developers - narginchk(3, Inf); + narginchk(2, Inf); - BIDS = bids.layout(BIDS); + if nargin < 3 + pipeline_name = 'bids-matlab'; + end - if ~exist(out_path, 'dir') - mkdir(out_path); + if nargin < 4 + filter = []; end - derivatives = []; - data_list = bids.query(BIDS, 'data', query); - subjects_list = bids.query(BIDS, 'subjects', query); + if nargin < 5 || isempty(unzip) + unzip = true; + end + + if nargin < 6 || isempty(force) + force = false; + end + + if nargin < 7 || isempty(verbose) + verbose = false; + end + + BIDS = bids.layout(BIDS); + + % Check that we actually have to copy something + data_list = bids.query(BIDS, 'data', filter); + subjects_list = bids.query(BIDS, 'subjects', filter); if isempty(data_list) warning('No data found for this query'); @@ -47,12 +72,19 @@ fprintf('Found %d files in %d subjects\n', length(data_list), length(subjects_list)); end + % Determine and create output directory + if nargin < 2 || isempty(out_path) + out_path = fullfile(BIDS.dir, '..', 'derivatives'); + end + if ~exist(out_path, 'dir') + mkdir(out_path); + end derivatives_folder = fullfile(out_path, pipeline_name); if ~exist(derivatives_folder, 'dir') mkdir(derivatives_folder); end - % creating / loading description + % Creating / loading description descr_file = fullfile(derivatives_folder, 'dataset_description.json'); if exist(descr_file, 'file') description = bids.util.jsondecode(descr_file); @@ -65,7 +97,7 @@ pipeline.Version = ''; pipeline.Container = ''; if isfield(description, 'GeneratedBy') - description.GeneratedBy = [description.GeneratedBy pipeline]; + description.GeneratedBy = [description.GeneratedBy; pipeline]; else description.GeneratedBy = pipeline; end @@ -76,12 +108,14 @@ % looping over selected files for iFile = 1:numel(data_list) - copy_file(BIDS, derivatives_folder, data_list{iFile}); + copy_file(BIDS, derivatives_folder, data_list{iFile}, unzip, force, verbose); end + %% + derivatives = []; end -function copy_file(BIDS, derivatives_folder, data_file) +function copy_file(BIDS, derivatives_folder, data_file, unzip, force, verbose) info = bids.internal.return_file_info(BIDS, data_file); file = BIDS.subjects(info.sub_idx).(info.modality)(info.file_idx); @@ -96,9 +130,15 @@ function copy_file(BIDS, derivatives_folder, data_file) %% ignore already existing files % avoid circular references - if exist(output_metadata_file, 'file') + if ~force && exist(output_metadata_file, 'file') + if verbose + fprintf(1, '\n skipping: %s', file.filename); + end return else + if verbose + fprintf(1, '\n copying: %s', file.filename); + end file.meta = bids.internal.get_metadata(file.metafile); end @@ -110,7 +150,7 @@ function copy_file(BIDS, derivatives_folder, data_file) % we follow any eventual symlink % and then unzip the data if necessary copy_with_symlink(data_file, fullfile(out_dir, file.filename)); - unzip_data(file, out_dir); + unzip_data(file, out_dir, unzip); %% export metadata % All the metadata of each file is read through the whole hierarchy @@ -131,7 +171,7 @@ function copy_file(BIDS, derivatives_folder, data_file) for ifile = 1:numel(file.dependencies.(dependencies{dep})) dep_file = file.dependencies.(dependencies{dep}){ifile}; if exist(dep_file, 'file') - copy_file(BIDS, derivatives_folder, dep_file); + copy_file(BIDS, derivatives_folder, dep_file, unzip, force, verbose); else warning(['Dependency file ' dep_file ' not found']); end @@ -146,6 +186,12 @@ function copy_with_symlink(src, target) % Follows symbolic link to copy data: % Might be necessary for datasets curated with datalad % + % Comment from Guillaume: + % I think we should make a system() call only out of necessity. + % We could test for symlinks within a isunix condition and only use cp -L for these? + % + % Though datalad should run on windows too + % command = 'cp -R -L -f'; @@ -179,7 +225,10 @@ function copy_with_symlink(src, target) end -function unzip_data(file, out_dir) +function unzip_data(file, out_dir, unzip) + if ~unzip + return + end % to ensure a consistent behavior with matlab and octave if bids.internal.ends_with(file.ext, '.gz') gunzip(fullfile(out_dir, file.filename)); diff --git a/+bids/query.m b/+bids/query.m index c73064f7..5bad63eb 100644 --- a/+bids/query.m +++ b/+bids/query.m @@ -126,6 +126,10 @@ elseif iscell(options{1}) options = options{1}; + + elseif isempty(options{1}) + options = cell(0, 2); + return end else diff --git a/tests/test_copy_to_derivative.m b/tests/test_copy_to_derivative.m index b47505f1..194ca283 100644 --- a/tests/test_copy_to_derivative.m +++ b/tests/test_copy_to_derivative.m @@ -6,6 +6,19 @@ initTestSuite; end +function test_copy_to_derivative_MoAE() + + input_dir = download_moae_ds(true()); + out_path = fullfile(input_dir, 'MoAEpilot', 'derivatives'); + + BIDS = fullfile(input_dir, 'MoAEpilot'); + + pipeline_name = 'bids-matlab'; + + derivatives = bids.copy_to_derivative(BIDS, out_path, pipeline_name); + +end + function test_copy_to_derivative_ds000001() input_dir = fullfile(pwd, 'data', 'ds000001'); @@ -17,26 +30,23 @@ function test_copy_to_derivative_ds000001() BIDS = fullfile(input_dir); - what_to_copy = struct('sub', '01', ... - 'modality', 'func', ... - 'suffix', 'bold'); - what_to_copy.run = {'01'; '03'}; - - pipeline_name = 'bids-matlab'; - - derivatives = bids.copy_to_derivative(BIDS, out_path, pipeline_name, what_to_copy); + filters = struct('sub', '01', ... + 'modality', 'func', ... + 'suffix', 'bold'); + filters.run = {'01'; '03'}; + + output_dir = []; + pipeline_name = []; + unzip = false; + force = false; + verbose = true; + + derivatives = bids.copy_to_derivative(BIDS, ... + output_dir, ... + pipeline_name, ... + filters, ... + unzip, ... + force, ... + verbose); end - -% function test_copy_to_derivative_MoAE() -% -% input_dir = download_moae_ds(true()); -% out_path = fullfile(input_dir, 'MoAEpilot', 'derivatives'); -% -% BIDS = fullfile(input_dir, 'MoAEpilot'); -% -% pipeline_name = 'bids-matlab'; -% -% derivatives = bids.copy_to_derivative(BIDS, out_path, pipeline_name); -% -% end From c7b3853ab32f429655f050826e2a40cb99ead46a Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 17 Apr 2021 11:48:56 +0200 Subject: [PATCH 30/36] move test data for copy_to_derivatives --- .github/workflows/run_tests.yml | 5 +++-- .gitignore | 6 +++++- tests/README.md | 2 +- tests/test_copy_to_derivative.m | 4 ++-- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 1eb61343..13b46979 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -48,10 +48,11 @@ jobs: cd tests git clone git://github.com/bids-standard/bids-examples.git --depth 1 octave $OCTFLAGS --eval "download_moae_ds" - cd data + cd .. + mkdir data; cd data datalad clone https://github.com/OpenNeuroDatasets/ds000001.git datalad get ds000001/sub-01/func - cd ../.. + cd .. - name: MOxUnit Action uses: joergbrech/moxunit-action@master diff --git a/.gitignore b/.gitignore index 64d4b1f1..6a956d4d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,11 @@ # ignore virtual env env/* -# ignore bids-specification repo that gets cloned during CI +# ignore bids-specification repo (necessary for convert_schema.py) bids-specification/* +# ignore input and output data involved in tests +data/* +derivatives/* + .prettierrc diff --git a/tests/README.md b/tests/README.md index ddd02eaa..4e85385a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -60,7 +60,7 @@ SPM website. To get more complex data sets, to test things you can use datalad. ```bash -cd tests/data +mkdir data datalad clone https://github.com/OpenNeuroDatasets/ds000001.git datalad get ds000001/sub-01/ ``` diff --git a/tests/test_copy_to_derivative.m b/tests/test_copy_to_derivative.m index 194ca283..cff0621d 100644 --- a/tests/test_copy_to_derivative.m +++ b/tests/test_copy_to_derivative.m @@ -21,8 +21,8 @@ function test_copy_to_derivative_MoAE() function test_copy_to_derivative_ds000001() - input_dir = fullfile(pwd, 'data', 'ds000001'); - out_path = fullfile(pwd, 'data', 'derivatives'); + input_dir = fullfile('..', 'data', 'ds000001'); + out_path = fullfile('..', 'data', 'derivatives'); if exist(out_path, 'dir') rmdir(out_path, 's'); From 997b07f878e8e492d9e41a05dd57f04cd9a3187c Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 17 Apr 2021 12:22:21 +0200 Subject: [PATCH 31/36] add test to check fmap dependencies are copied --- +bids/copy_to_derivative.m | 31 ++++++++++++++++------ .github/workflows/run_tests.yml | 3 +++ tests/test_copy_to_derivative.m | 46 +++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/+bids/copy_to_derivative.m b/+bids/copy_to_derivative.m index 1cff3ca9..76c1f2a9 100644 --- a/+bids/copy_to_derivative.m +++ b/+bids/copy_to_derivative.m @@ -1,4 +1,4 @@ -function derivatives = copy_to_derivative(BIDS, out_path, pipeline_name, filter, unzip, force, verbose) +function derivatives = copy_to_derivative(BIDS, out_path, pipeline_name, filter, unzip, force, skip_dep, use_schema, verbose) % % Copy selected data from BIDS layout to given derivatives folder, % returning layout of new derivatives folder @@ -55,11 +55,21 @@ force = false; end - if nargin < 7 || isempty(verbose) + if nargin < 7 || isempty(skip_dep) + skip_dep = false; + end + + if nargin < 8 || isempty(use_schema) + use_schema = true; + end + + if nargin < 9 || isempty(verbose) verbose = false; end - BIDS = bids.layout(BIDS); + derivatives = []; + + BIDS = bids.layout(BIDS, use_schema); % Check that we actually have to copy something data_list = bids.query(BIDS, 'data', filter); @@ -108,14 +118,14 @@ % looping over selected files for iFile = 1:numel(data_list) - copy_file(BIDS, derivatives_folder, data_list{iFile}, unzip, force, verbose); + copy_file(BIDS, derivatives_folder, data_list{iFile}, unzip, force, skip_dep, verbose); end %% - derivatives = []; + derivatives = bids.layout(derivatives_folder, use_schema); end -function copy_file(BIDS, derivatives_folder, data_file, unzip, force, verbose) +function copy_file(BIDS, derivatives_folder, data_file, unzip, force, skip_dep, verbose) info = bids.internal.return_file_info(BIDS, data_file); file = BIDS.subjects(info.sub_idx).(info.modality)(info.file_idx); @@ -165,18 +175,23 @@ function copy_file(BIDS, derivatives_folder, data_file, unzip, force, verbose) end %% dealing with dependencies - if ~isempty(file.dependencies) + if ~skip_dep + dependencies = fieldnames(file.dependencies); + for dep = 1:numel(dependencies) for ifile = 1:numel(file.dependencies.(dependencies{dep})) + dep_file = file.dependencies.(dependencies{dep}){ifile}; if exist(dep_file, 'file') - copy_file(BIDS, derivatives_folder, dep_file, unzip, force, verbose); + copy_file(BIDS, derivatives_folder, dep_file, unzip, force, skip_dep, verbose); else warning(['Dependency file ' dep_file ' not found']); end + end end + end end diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 13b46979..2794ae29 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -52,6 +52,9 @@ jobs: mkdir data; cd data datalad clone https://github.com/OpenNeuroDatasets/ds000001.git datalad get ds000001/sub-01/func + datalad clone https://github.com/OpenNeuroDatasets/ds000117.git + datalad get ds000117/sub-01/ses-mri/func/*run-0[13]* + datalad get ds000117/sub-01/ses-mri/fmap/ cd .. - name: MOxUnit Action diff --git a/tests/test_copy_to_derivative.m b/tests/test_copy_to_derivative.m index cff0621d..472603c1 100644 --- a/tests/test_copy_to_derivative.m +++ b/tests/test_copy_to_derivative.m @@ -39,6 +39,8 @@ function test_copy_to_derivative_ds000001() pipeline_name = []; unzip = false; force = false; + skip_dependencies = true; + use_schema = true; verbose = true; derivatives = bids.copy_to_derivative(BIDS, ... @@ -47,6 +49,50 @@ function test_copy_to_derivative_ds000001() filters, ... unzip, ... force, ... + skip_dependencies, ... + use_schema, ... verbose); + copied_files = bids.query(derivatives, 'data'); + assertEqual(size(copied_files, 1), 2); + +end + +function test_copy_to_derivative_ds000117() + + input_dir = fullfile('..', 'data', 'ds000117'); + out_path = fullfile('..', 'data', 'derivatives'); + + if exist(out_path, 'dir') + rmdir(out_path, 's'); + end + + BIDS = fullfile(input_dir); + + filters = struct('sub', '01', ... + 'modality', 'func', ... + 'suffix', 'bold'); + filters.run = {'01'; '03'}; + + output_dir = []; + pipeline_name = []; + unzip = false; + force = false; + skip_dependencies = false; + use_schema = false; + verbose = true; + + derivatives = bids.copy_to_derivative(BIDS, ... + output_dir, ... + pipeline_name, ... + filters, ... + unzip, ... + force, ... + skip_dependencies, ... + use_schema, ... + verbose); + + copied_files = bids.query(derivatives, 'data'); + assertEqual(size(copied_files, 1), 13); + end From 9581fa510f362c29f5a19b7a7daf7cc00a3e9e3e Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 17 Apr 2021 12:38:25 +0200 Subject: [PATCH 32/36] simplify tests copy derivatives use bids examples by skipping the unzipping --- .github/workflows/run_tests.yml | 16 +++++---- tests/README.md | 6 ++++ tests/test_copy_to_derivative.m | 57 +++++++++++++-------------------- 3 files changed, 38 insertions(+), 41 deletions(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 2794ae29..47a6b4bb 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -49,13 +49,15 @@ jobs: git clone git://github.com/bids-standard/bids-examples.git --depth 1 octave $OCTFLAGS --eval "download_moae_ds" cd .. - mkdir data; cd data - datalad clone https://github.com/OpenNeuroDatasets/ds000001.git - datalad get ds000001/sub-01/func - datalad clone https://github.com/OpenNeuroDatasets/ds000117.git - datalad get ds000117/sub-01/ses-mri/func/*run-0[13]* - datalad get ds000117/sub-01/ses-mri/fmap/ - cd .. + + # if tests on real data are needed + # mkdir data; cd data + # datalad clone https://github.com/OpenNeuroDatasets/ds000001.git + # datalad get ds000001/sub-01/func + # datalad clone https://github.com/OpenNeuroDatasets/ds000117.git + # datalad get ds000117/sub-01/ses-mri/func/*run-0[13]* + # datalad get ds000117/sub-01/ses-mri/fmap/ + # cd .. - name: MOxUnit Action uses: joergbrech/moxunit-action@master diff --git a/tests/README.md b/tests/README.md index 4e85385a..3f3e95c9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -63,6 +63,12 @@ To get more complex data sets, to test things you can use datalad. mkdir data datalad clone https://github.com/OpenNeuroDatasets/ds000001.git datalad get ds000001/sub-01/ + +datalad clone https://github.com/OpenNeuroDatasets/ds000117.git +datalad get ds000117/sub-01/ses-mri/func +datalad get ds000117/sub-01/ses-mri/fmap + + ``` ## Add helper functions to the path diff --git a/tests/test_copy_to_derivative.m b/tests/test_copy_to_derivative.m index 472603c1..2358878b 100644 --- a/tests/test_copy_to_derivative.m +++ b/tests/test_copy_to_derivative.m @@ -9,7 +9,7 @@ function test_copy_to_derivative_MoAE() input_dir = download_moae_ds(true()); - out_path = fullfile(input_dir, 'MoAEpilot', 'derivatives'); + out_path = []; BIDS = fullfile(input_dir, 'MoAEpilot'); @@ -19,10 +19,17 @@ function test_copy_to_derivative_MoAE() end -function test_copy_to_derivative_ds000001() +function test_copy_to_derivative_ds000117() - input_dir = fullfile('..', 'data', 'ds000001'); - out_path = fullfile('..', 'data', 'derivatives'); + pth_bids_example = get_test_data_dir(); + input_dir = fullfile(pth_bids_example, 'ds000117'); + + % to test on real data uncomment the following line + % see tests/README.md to see how to install the data + % + % input_dir = fullfile('..', 'data', 'ds000117'); + + out_path = fullfile(pwd, 'data', 'derivatives'); if exist(out_path, 'dir') rmdir(out_path, 's'); @@ -35,16 +42,16 @@ function test_copy_to_derivative_ds000001() 'suffix', 'bold'); filters.run = {'01'; '03'}; - output_dir = []; + %% pipeline_name = []; unzip = false; force = false; - skip_dependencies = true; - use_schema = true; + skip_dependencies = false; + use_schema = false; verbose = true; derivatives = bids.copy_to_derivative(BIDS, ... - output_dir, ... + out_path, ... pipeline_name, ... filters, ... unzip, ... @@ -54,36 +61,16 @@ function test_copy_to_derivative_ds000001() verbose); copied_files = bids.query(derivatives, 'data'); - assertEqual(size(copied_files, 1), 2); - -end - -function test_copy_to_derivative_ds000117() - - input_dir = fullfile('..', 'data', 'ds000117'); - out_path = fullfile('..', 'data', 'derivatives'); - + assertEqual(size(copied_files, 1), 13); + + %% same but we skip dependencies if exist(out_path, 'dir') rmdir(out_path, 's'); end - - BIDS = fullfile(input_dir); - - filters = struct('sub', '01', ... - 'modality', 'func', ... - 'suffix', 'bold'); - filters.run = {'01'; '03'}; - - output_dir = []; - pipeline_name = []; - unzip = false; - force = false; - skip_dependencies = false; - use_schema = false; - verbose = true; + skip_dependencies = true; derivatives = bids.copy_to_derivative(BIDS, ... - output_dir, ... + out_path, ... pipeline_name, ... filters, ... unzip, ... @@ -93,6 +80,8 @@ function test_copy_to_derivative_ds000117() verbose); copied_files = bids.query(derivatives, 'data'); - assertEqual(size(copied_files, 1), 13); + assertEqual(size(copied_files, 1), 4); + + %% add test to check that only files that conform to schema are copied end From 0521460788cc3849aeceaf6fec411012432795f4 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 17 Apr 2021 13:59:12 +0200 Subject: [PATCH 33/36] move download moae function --- .github/workflows/run_tests.yml | 13 +++++-------- tests/test_copy_to_derivative.m | 14 +++++++------- tests/{ => utils}/download_moae_ds.m | 0 3 files changed, 12 insertions(+), 15 deletions(-) rename tests/{ => utils}/download_moae_ds.m (100%) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 47a6b4bb..b1b987dc 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -23,8 +23,6 @@ jobs: run: | sudo apt-get -y -qq update sudo apt-get -y install octave liboctave-dev - sudo apt-get -y install nodejs npm - sudo apt-get -y install datalad - name: Clone bids-matlab uses: actions/checkout@v2 @@ -32,10 +30,6 @@ jobs: submodules: true fetch-depth: 2 - - name: Install BIDS validator - run: | - npm install `cat npm-requirements.txt` - - name: Install JSONio run: | git clone git://github.com/gllmflndn/JSONio.git --depth 1 @@ -50,7 +44,10 @@ jobs: octave $OCTFLAGS --eval "download_moae_ds" cd .. - # if tests on real data are needed + # ------------------------------ + # if tests on real data are needed + # + # sudo apt-get -y install datalad # mkdir data; cd data # datalad clone https://github.com/OpenNeuroDatasets/ds000001.git # datalad get ds000001/sub-01/func @@ -63,7 +60,7 @@ jobs: uses: joergbrech/moxunit-action@master with: tests: tests # files or directories containing the MOxUnit test cases - src: +bids # directories to be added to the Octave search path before running the tests. + src: +bids # directories to be added to the Octave search path before running the tests. ext: JSONio tests/utils # External resources to add to the search put (excluded from coverage) # data: # Directory for test data with_coverage: true diff --git a/tests/test_copy_to_derivative.m b/tests/test_copy_to_derivative.m index 2358878b..b5bf7a62 100644 --- a/tests/test_copy_to_derivative.m +++ b/tests/test_copy_to_derivative.m @@ -21,14 +21,14 @@ function test_copy_to_derivative_MoAE() function test_copy_to_derivative_ds000117() - pth_bids_example = get_test_data_dir(); - input_dir = fullfile(pth_bids_example, 'ds000117'); - + pth_bids_example = get_test_data_dir(); + input_dir = fullfile(pth_bids_example, 'ds000117'); + % to test on real data uncomment the following line % see tests/README.md to see how to install the data % % input_dir = fullfile('..', 'data', 'ds000117'); - + out_path = fullfile(pwd, 'data', 'derivatives'); if exist(out_path, 'dir') @@ -62,7 +62,7 @@ function test_copy_to_derivative_ds000117() copied_files = bids.query(derivatives, 'data'); assertEqual(size(copied_files, 1), 13); - + %% same but we skip dependencies if exist(out_path, 'dir') rmdir(out_path, 's'); @@ -80,8 +80,8 @@ function test_copy_to_derivative_ds000117() verbose); copied_files = bids.query(derivatives, 'data'); - assertEqual(size(copied_files, 1), 4); - + assertEqual(size(copied_files, 1), 4); + %% add test to check that only files that conform to schema are copied end diff --git a/tests/download_moae_ds.m b/tests/utils/download_moae_ds.m similarity index 100% rename from tests/download_moae_ds.m rename to tests/utils/download_moae_ds.m From 7a242eaa324fbad519691668d7bd5628c49c1f52 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 17 Apr 2021 14:29:41 +0200 Subject: [PATCH 34/36] update github action --- .github/workflows/run_tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index b1b987dc..0c137fc3 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -41,8 +41,9 @@ jobs: run: | cd tests git clone git://github.com/bids-standard/bids-examples.git --depth 1 + cd util octave $OCTFLAGS --eval "download_moae_ds" - cd .. + cd ../.. # ------------------------------ # if tests on real data are needed From b425fdb17cc8077ae2f7b794912f9978f042eab9 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 17 Apr 2021 14:53:48 +0200 Subject: [PATCH 35/36] update download moae --- tests/utils/download_moae_ds.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils/download_moae_ds.m b/tests/utils/download_moae_ds.m index 6f865420..fd3caa35 100644 --- a/tests/utils/download_moae_ds.m +++ b/tests/utils/download_moae_ds.m @@ -9,7 +9,7 @@ downloadData = true(); end - output_dir = fullfile(fileparts(mfilename('fullpath')), 'data'); + output_dir = fullfile(fileparts(mfilename('fullpath')), '..', 'data'); if downloadData From 63a9b1b2c36633abf24c54a46d7c662a9c04ac0b Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sat, 17 Apr 2021 15:24:11 +0200 Subject: [PATCH 36/36] fix typo --- .github/workflows/run_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 0c137fc3..d5f740e3 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -41,7 +41,7 @@ jobs: run: | cd tests git clone git://github.com/bids-standard/bids-examples.git --depth 1 - cd util + cd utils octave $OCTFLAGS --eval "download_moae_ds" cd ../..