From d06b845e974b67ee30ecdbba1338aa72ce1f1a00 Mon Sep 17 00:00:00 2001 From: Mark Henney Date: Tue, 13 Feb 2024 15:23:52 +0100 Subject: [PATCH 1/8] completed raw1 curration --- data_curration/convert_source_to_raw1.m | 60 +++++++++++++++++++------ 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/data_curration/convert_source_to_raw1.m b/data_curration/convert_source_to_raw1.m index 121e802..5b8a70a 100644 --- a/data_curration/convert_source_to_raw1.m +++ b/data_curration/convert_source_to_raw1.m @@ -52,11 +52,13 @@ %% Add util dir to path addpath('/project/3031004.01/meg-ahat/util') -%% Setup Fieldtrip -configure_ft - %% Set pilot data directory data_dir = '/project/3031004.01/pilot-data'; +diaryfile = strcat(data_dir, '/source_to_raw1.log'); +diary (diaryfile) + +%% Setup Fieldtrip +configure_ft % Sebject (defined as cell array for future compatibility) subj = { @@ -65,7 +67,7 @@ % Tasks in the two runs tasks = { - 'visual'; + 'visualattention'; 'workingmemory' }; @@ -112,15 +114,18 @@ % HEAD POSITION (Polhemus) % % Specify source file path - head_pos_file = sprintf('sub-%s.pos', subj{subindx}) + head_pos_file = sprintf('sub-%s.pos', subj{subindx}); source_ses1_dir = sprintf('%s/source/sub-%s/ses-001/', data_dir, subj{subindx}); head_pos_d = strcat(source_ses1_dir, head_pos_file); % Specify destination file path - head_pos_file_raw1 = sprintf('sub-%s_ses-001_headshape.pos', subj{subindx}) + head_pos_file_raw1 = sprintf('sub-%s_ses-001_headshape.pos', subj{subindx}); source_ses1_raw1_dir = sprintf('%s/sub-%s/ses-001/meg/', general_cfg.bidsroot, subj{subindx}); head_pos_raw1_d = strcat(source_ses1_raw1_dir, head_pos_file_raw1); + % Check that destination folder exists + mkdir (source_ses1_raw1_dir) + % Copy source to raw1 destination copyfile(head_pos_d, head_pos_raw1_d); @@ -157,6 +162,11 @@ cfg.dataset_description.BIDSVersion = 'v1.5.0'; cfg.TaskDescription = 'The experiment consisted of visual flicker stimulation combined with a visual attention discrimination task and an arithmetic task.'; cfg.task = 'flicker'; + cfg.PowerLineFrequency = 50; + cfg.DewarPosition = 'upright'; + cfg.SoftwareFilters = 'n/a'; + cfg.DigitizedLandmarks = true; + cfg.DigitizedHeadPoints = false; % Convert to BIDS in raw1 try @@ -178,8 +188,8 @@ cfg.dataset_description.BIDSVersion = 'unofficial extension'; cfg.method = 'convert'; % the eyelink-specific format is not supported, convert it to plain TSV cfg.dataset = fullfile(d.folder, d.name); - cfg.datatype = 'eyetracker'; - cfg.suffix = 'meg'; + cfg.datatype = 'eyetrack'; + cfg.suffix = 'eyetrack'; cfg.Manufacturer = 'SR Research'; cfg.ManufacturerModelName = 'Eyelink 1000'; cfg.TaskDescription = 'The experiment consisted of visual flicker stimulation combined with a visual attention discrimination task and an arithmetic task.'; @@ -193,6 +203,12 @@ % % Iterate over runs % + % First, create a /beh/ dir + beh_dir = sprintf('%s/sub-%s/ses-00%d/beh/', general_cfg.bidsroot, subj{subindx}, sesindx); + if not(isfolder(beh_dir)) + mkdir(beh_dir) + end + for runindx=1:2 % % BEHAVIOURAL DATA @@ -205,11 +221,17 @@ d = dir(strcat(behav_dir, filename)); % Load the behavioural data as a table log = readtable(fullfile(d.folder, d.name)); + + % Add BIDS required columns + log_len = size(log, 1); + log.onset = repmat("n/a", log_len, 1); + log.duration = repmat("n/a", log_len, 1); + log = log(:, [end-1:end 1:end-3]); % Specify the output path in the raw1 directory - out_dir = sprintf('%s/sub-%s/ses-00%d/beh/', general_cfg.bidsroot, subj{subindx}, sesindx); - out_file = sprintf('sub-%s_ses-00%d_run-00%d_beh.tsv', subj{subindx}, sesindx, runindx); - out_d = fullfile(strcat(out_dir, out_file)); + filename = sprintf('sub-%s_ses-00%d_run-00%d_task-%s_events', subj{subindx}, sesindx, runindx, tasks{runindx}); + out_file = strcat(filename, '.tsv'); + out_d = fullfile(strcat(beh_dir, out_file)); % Write the behavioural data to destination writetable(log,out_d, 'filetype','text', 'delimiter','\t') @@ -236,8 +258,8 @@ json_txt = jsonencode(info, "PrettyPrint", true); % Specify the destination - out_json_file = sprintf('sub-%s_ses-00%d_run-00%d_beh.json', subj{subindx}, sesindx, runindx); - out_json_d = fullfile(strcat(out_dir, out_json_file)); + out_json_file = strcat(filename, '.json'); + out_json_d = fullfile(strcat(beh_dir, out_json_file)); % Write the file fid = fopen(out_json_d,'w'); fprintf(fid,'%s',json_txt); @@ -245,4 +267,14 @@ end % for each run end % for each ses -end % for each subject \ No newline at end of file +end % for each subject + +% Write entries to the .bidsignore file +bidsignore_text = ['*eyetrack.tsv' newline '*eyetrack.json' newline]; +% Specify the destination +bidsignore_file = fullfile(strcat(general_cfg.bidsroot, '/.bidsignore')); +% Write the file +fid = fopen(bidsignore_file, 'w'); +fprintf(fid, '%s', bidsignore_text); + +diary off \ No newline at end of file From 253d511e6e7016a6b891e949bd0c525d7032a980 Mon Sep 17 00:00:00 2001 From: Mark Henney Date: Tue, 13 Feb 2024 15:33:00 +0100 Subject: [PATCH 2/8] Updated tree in comments --- data_curration/convert_source_to_raw1.m | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data_curration/convert_source_to_raw1.m b/data_curration/convert_source_to_raw1.m index 5b8a70a..2a4d8b2 100644 --- a/data_curration/convert_source_to_raw1.m +++ b/data_curration/convert_source_to_raw1.m @@ -23,10 +23,12 @@ % | |-- ses-001 % | | |-- anat % | | |-- beh +% | | |-- eyetrack % | | |-- meg % | | `-- sub-099_ses-001_scans.tsv % | |-- ses-002 % | | |-- beh +% | | |-- eyetrack % | | |-- meg % | | `-- sub-099_ses-002_scans.tsv % | `-- sub-099_sessions.tsv From 9b9dcb87e3125e999ab097bf4c049f6ceb51fa78 Mon Sep 17 00:00:00 2001 From: Mark Henney Date: Tue, 13 Feb 2024 15:38:51 +0100 Subject: [PATCH 3/8] updated tree and link to github --- documentation/DATA_MANAGEMENT.stub | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/documentation/DATA_MANAGEMENT.stub b/documentation/DATA_MANAGEMENT.stub index 118348b..40d8a33 100644 --- a/documentation/DATA_MANAGEMENT.stub +++ b/documentation/DATA_MANAGEMENT.stub @@ -7,6 +7,8 @@ this project aims to conduct data analysis on BIDS-compliant data from the get-g the source data is converted to BIDS prior to data analysis. However, the conversion takes place in two steps to incrementally enrich the data structure. +Data analysis is conducted solely on the completely enriched data set (raw2 below). + ### Data Versions The project pilot-data exists in several versions that can all be accessed in the MEG-AHAT @@ -25,7 +27,7 @@ include: ### Converting Data Source data is converted to raw1 using the -[`/data_curration/convert_source_to_raw1.m`](data_curration/convert_source_to_raw1.m) script. +[`/data_curration/convert_source_to_raw1.m`](https://github.com/henneysq/meg-ahat/blob/main/data_curration/convert_source_to_raw1.m) script. Raw1 data is converted to raw2 using the [`/data_curration/convert_raw1_to_raw2.m`](data_curration/convert_raw1_to_raw2.m) script. @@ -41,10 +43,12 @@ pilot-data | |-- ses-001 | | |-- anat | | |-- beh +| | |-- eyetrack | | |-- meg | | `-- sub-099_ses-001_scans.tsv | |-- ses-002 | | |-- beh +| | |-- eyetrack | | |-- meg | | `-- sub-099_ses-002_scans.tsv | `-- sub-099_sessions.tsv @@ -65,3 +69,7 @@ pilot-data |-- sub-099_ses-002-eyetracking.asc `-- sub-099_ses-002-eyetracking.edf ``` + +### Eyetrack data + +Currently, eyetracking data is not covered by BIDS, but is added with the [BIDS extension proposal BEP020](https://docs.google.com/document/d/1eggzTCzSHG3AEKhtnEDbcdk-2avXN6I94X8aUPEBVsw/edit#heading=h.9tphvz6ot0j1). \ No newline at end of file From 77c39e1a716abf539919ea5a9d1d9395bde3a0ab Mon Sep 17 00:00:00 2001 From: Github Action Date: Tue, 13 Feb 2024 14:39:10 +0000 Subject: [PATCH 4/8] Updated README --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 327a3b4..1178a98 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ See also the [`DCCN project proposal slides`](protocol/Invisible-Flicker_aka_MEG 1. [Philosophy](#philosophy) 2. [Data Versions](#data-versions) 3. [Converting Data](#converting-data) + 4. [Eyetrack data](#eyetrack-data) 3. [Experiment Management](#experiment-management) 1. [Dependencies](#dependencies) 2. [Quick-Start Guide](#quick-start-guide) @@ -40,6 +41,8 @@ this project aims to conduct data analysis on BIDS-compliant data from the get-g the source data is converted to BIDS prior to data analysis. However, the conversion takes place in two steps to incrementally enrich the data structure. +Data analysis is conducted solely on the completely enriched data set (raw2 below). + ### Data Versions The project pilot-data exists in several versions that can all be accessed in the MEG-AHAT @@ -58,7 +61,7 @@ include: ### Converting Data Source data is converted to raw1 using the -[`/data_curration/convert_source_to_raw1.m`](data_curration/convert_source_to_raw1.m) script. +[`/data_curration/convert_source_to_raw1.m`](https://github.com/henneysq/meg-ahat/blob/main/data_curration/convert_source_to_raw1.m) script. Raw1 data is converted to raw2 using the [`/data_curration/convert_raw1_to_raw2.m`](data_curration/convert_raw1_to_raw2.m) script. @@ -74,10 +77,12 @@ pilot-data | |-- ses-001 | | |-- anat | | |-- beh +| | |-- eyetrack | | |-- meg | | `-- sub-099_ses-001_scans.tsv | |-- ses-002 | | |-- beh +| | |-- eyetrack | | |-- meg | | `-- sub-099_ses-002_scans.tsv | `-- sub-099_sessions.tsv @@ -99,6 +104,9 @@ pilot-data `-- sub-099_ses-002-eyetracking.edf ``` +### Eyetrack data + +Currently, eyetracking data is not covered by BIDS, but is added with the [BIDS extension proposal BEP020](https://docs.google.com/document/d/1eggzTCzSHG3AEKhtnEDbcdk-2avXN6I94X8aUPEBVsw/edit#heading=h.9tphvz6ot0j1). ## Experiment Management From 1bea82bef53ca11b4a00fc26f7db0d2e316e9f95 Mon Sep 17 00:00:00 2001 From: Mark Henney Date: Tue, 13 Feb 2024 15:49:00 +0100 Subject: [PATCH 5/8] update data management documentation --- documentation/DATA_MANAGEMENT.stub | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/documentation/DATA_MANAGEMENT.stub b/documentation/DATA_MANAGEMENT.stub index 40d8a33..2191ced 100644 --- a/documentation/DATA_MANAGEMENT.stub +++ b/documentation/DATA_MANAGEMENT.stub @@ -19,19 +19,30 @@ include: respective recording devices and includes MEG scans, simultaneous eye-tracking and presentation log-files, polhemous positional data, and MRI scans. 2. First iteration raw data (`/raw1/`) contains a BIDS-compliant version with minimal - changes + changes. One implication of this is that events are not yet alligned, as the recorded + behavioural data is not set on a time axis at the point of recording. 2. Second iteration raw data (`/raw2/`) contains a BIDS-compliant version of the data in which the behevioural data recorded along with MEG has been moved to the /meg - directory, and files will are enriched and aligned in time. + directory, and MEG, eyetrack, and behavioural events are enriched and aligned in time. ### Converting Data +Converting data between versions is non-trivial and requires both commonly used public +tools and custom scripts. This projects uses the FieldTrip tool (`data2bids`)[https://github.com/fieldtrip/fieldtrip/blob/master/data2bids.m] +and custom matlab scripts. + Source data is converted to raw1 using the [`/data_curration/convert_source_to_raw1.m`](https://github.com/henneysq/meg-ahat/blob/main/data_curration/convert_source_to_raw1.m) script. Raw1 data is converted to raw2 using the [`/data_curration/convert_raw1_to_raw2.m`](data_curration/convert_raw1_to_raw2.m) script. +#### Eyetrack data + +Currently, eyetracking data is not covered by BIDS, but is added with the [BIDS extension proposal BEP020](https://docs.google.com/document/d/1eggzTCzSHG3AEKhtnEDbcdk-2avXN6I94X8aUPEBVsw/edit#heading=h.9tphvz6ot0j1). + +With this change, eyetracker data is allocated to the `eyetrack/` scans directory and has the `_eyetrack` suffix. + #### Directory Layout ``` @@ -68,8 +79,4 @@ pilot-data |-- pilot002_3031000.01_20231214_01.ds |-- sub-099_ses-002-eyetracking.asc `-- sub-099_ses-002-eyetracking.edf -``` - -### Eyetrack data - -Currently, eyetracking data is not covered by BIDS, but is added with the [BIDS extension proposal BEP020](https://docs.google.com/document/d/1eggzTCzSHG3AEKhtnEDbcdk-2avXN6I94X8aUPEBVsw/edit#heading=h.9tphvz6ot0j1). \ No newline at end of file +``` \ No newline at end of file From 2d77f4ea9454d45f7376ddb367a33369ff6f3750 Mon Sep 17 00:00:00 2001 From: Github Action Date: Tue, 13 Feb 2024 14:49:19 +0000 Subject: [PATCH 6/8] Updated README --- README.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1178a98..c97c7eb 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,6 @@ See also the [`DCCN project proposal slides`](protocol/Invisible-Flicker_aka_MEG 1. [Philosophy](#philosophy) 2. [Data Versions](#data-versions) 3. [Converting Data](#converting-data) - 4. [Eyetrack data](#eyetrack-data) 3. [Experiment Management](#experiment-management) 1. [Dependencies](#dependencies) 2. [Quick-Start Guide](#quick-start-guide) @@ -53,19 +52,30 @@ include: respective recording devices and includes MEG scans, simultaneous eye-tracking and presentation log-files, polhemous positional data, and MRI scans. 2. First iteration raw data (`/raw1/`) contains a BIDS-compliant version with minimal - changes + changes. One implication of this is that events are not yet alligned, as the recorded + behavioural data is not set on a time axis at the point of recording. 2. Second iteration raw data (`/raw2/`) contains a BIDS-compliant version of the data in which the behevioural data recorded along with MEG has been moved to the /meg - directory, and files will are enriched and aligned in time. + directory, and MEG, eyetrack, and behavioural events are enriched and aligned in time. ### Converting Data +Converting data between versions is non-trivial and requires both commonly used public +tools and custom scripts. This projects uses the FieldTrip tool (`data2bids`)[https://github.com/fieldtrip/fieldtrip/blob/master/data2bids.m] +and custom matlab scripts. + Source data is converted to raw1 using the [`/data_curration/convert_source_to_raw1.m`](https://github.com/henneysq/meg-ahat/blob/main/data_curration/convert_source_to_raw1.m) script. Raw1 data is converted to raw2 using the [`/data_curration/convert_raw1_to_raw2.m`](data_curration/convert_raw1_to_raw2.m) script. +#### Eyetrack data + +Currently, eyetracking data is not covered by BIDS, but is added with the [BIDS extension proposal BEP020](https://docs.google.com/document/d/1eggzTCzSHG3AEKhtnEDbcdk-2avXN6I94X8aUPEBVsw/edit#heading=h.9tphvz6ot0j1). + +With this change, eyetracker data is allocated to the `eyetrack/` scans directory and has the `_eyetrack` suffix. + #### Directory Layout ``` @@ -104,10 +114,6 @@ pilot-data `-- sub-099_ses-002-eyetracking.edf ``` -### Eyetrack data - -Currently, eyetracking data is not covered by BIDS, but is added with the [BIDS extension proposal BEP020](https://docs.google.com/document/d/1eggzTCzSHG3AEKhtnEDbcdk-2avXN6I94X8aUPEBVsw/edit#heading=h.9tphvz6ot0j1). - ## Experiment Management ### Dependencies From 81420e31d16d74ffcd65b8d9ddac2343d0e8c167 Mon Sep 17 00:00:00 2001 From: Mark Henney Date: Tue, 13 Feb 2024 15:52:33 +0100 Subject: [PATCH 7/8] fix hyperlinks --- documentation/DATA_MANAGEMENT.stub | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/DATA_MANAGEMENT.stub b/documentation/DATA_MANAGEMENT.stub index 2191ced..3a9b29f 100644 --- a/documentation/DATA_MANAGEMENT.stub +++ b/documentation/DATA_MANAGEMENT.stub @@ -28,14 +28,14 @@ include: ### Converting Data Converting data between versions is non-trivial and requires both commonly used public -tools and custom scripts. This projects uses the FieldTrip tool (`data2bids`)[https://github.com/fieldtrip/fieldtrip/blob/master/data2bids.m] +tools and custom scripts. This projects uses the FieldTrip tool [`data2bids`](https://github.com/fieldtrip/fieldtrip/blob/master/data2bids.m) and custom matlab scripts. Source data is converted to raw1 using the [`/data_curration/convert_source_to_raw1.m`](https://github.com/henneysq/meg-ahat/blob/main/data_curration/convert_source_to_raw1.m) script. Raw1 data is converted to raw2 using the -[`/data_curration/convert_raw1_to_raw2.m`](data_curration/convert_raw1_to_raw2.m) script. +[`/data_curration/convert_raw1_to_raw2.m`](https://github.com/henneysq/meg-ahat/blob/main/data_curration/convert_raw1_to_raw2.m) script. #### Eyetrack data From 2c6e26d4db9cb150ce32dd5554b2313554ba03b3 Mon Sep 17 00:00:00 2001 From: Github Action Date: Tue, 13 Feb 2024 14:52:49 +0000 Subject: [PATCH 8/8] Updated README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c97c7eb..a52111f 100644 --- a/README.md +++ b/README.md @@ -61,14 +61,14 @@ include: ### Converting Data Converting data between versions is non-trivial and requires both commonly used public -tools and custom scripts. This projects uses the FieldTrip tool (`data2bids`)[https://github.com/fieldtrip/fieldtrip/blob/master/data2bids.m] +tools and custom scripts. This projects uses the FieldTrip tool [`data2bids`](https://github.com/fieldtrip/fieldtrip/blob/master/data2bids.m) and custom matlab scripts. Source data is converted to raw1 using the [`/data_curration/convert_source_to_raw1.m`](https://github.com/henneysq/meg-ahat/blob/main/data_curration/convert_source_to_raw1.m) script. Raw1 data is converted to raw2 using the -[`/data_curration/convert_raw1_to_raw2.m`](data_curration/convert_raw1_to_raw2.m) script. +[`/data_curration/convert_raw1_to_raw2.m`](https://github.com/henneysq/meg-ahat/blob/main/data_curration/convert_raw1_to_raw2.m) script. #### Eyetrack data