Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Minor updates to the Jupyter notebooks #328

Merged
merged 3 commits into from
Oct 24, 2023
Merged

Minor updates to the Jupyter notebooks #328

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
84167cf
Merge pull request #326 from hed-standard/develop
VisLab Oct 17, 2023
e63986f
Merge branch 'main' of https://github.com/hed-standard/hed-examples i…
VisLab Oct 24, 2023
642dd60
Minor tweaks to jupyter notebooks
VisLab Oct 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 44 additions & 72 deletions hedcode/jupyter_notebooks/bids/find_event_combinations.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,77 +33,46 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 3,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" key_counts event_code cond_code event_type focus_modality \\\n",
"0 96 1 1 hear_word auditory \n",
"1 96 1 2 hear_word visual \n",
"2 288 1 3 hear_word auditory \n",
"3 96 2 1 look_word auditory \n",
"4 96 2 2 look_word visual \n",
"5 287 2 3 look_word visual \n",
"6 192 3 1 high_tone auditory \n",
"7 192 3 2 high_tone visual \n",
"8 192 4 1 light_bar auditory \n",
"9 192 4 2 light_bar visual \n",
"10 767 5 1 low_tone auditory \n",
"11 767 5 2 low_tone visual \n",
"12 766 6 1 dark_bar auditory \n",
"13 767 6 2 dark_bar visual \n",
"14 385 7 3 high_tone auditory \n",
"15 384 8 3 light_bar visual \n",
"16 191 9 3 high_tone visual \n",
"17 192 10 3 light_bar auditory \n",
"18 1531 11 3 low_tone auditory \n",
"19 1525 12 3 dark_bar visual \n",
"20 771 13 3 low_tone visual \n",
"21 774 14 3 dark_bar auditory \n",
"22 195 201 1 button_press auditory \n",
"23 191 201 2 button_press visual \n",
"24 390 201 3 button_press auditory \n",
"25 381 201 3 button_press visual \n",
"26 5 202 1 pause_recording auditory \n",
"27 6 202 2 pause_recording visual \n",
"28 6 202 3 pause_recording auditory \n",
"29 1 202 3 pause_recording nan \n",
"30 8 202 3 pause_recording visual \n",
"\n",
" attention_status task_role condition \n",
"0 unattended cue_auditory attend_auditory \n",
"1 unattended cue_visual attend_visual \n",
"2 attended cue_auditory shift_attention \n",
"3 unattended cue_auditory attend_auditory \n",
"4 unattended cue_visual attend_visual \n",
"5 attended cue_visual shift_attention \n",
"6 attended infrequent_stimulus attend_auditory \n",
"7 unattended infrequent_stimulus attend_visual \n",
"8 unattended infrequent_stimulus attend_auditory \n",
"9 attended infrequent_stimulus attend_visual \n",
"10 attended frequent_stimulus attend_auditory \n",
"11 unattended frequent_stimulus attend_visual \n",
"12 unattended frequent_stimulus attend_auditory \n",
"13 attended frequent_stimulus attend_visual \n",
"14 attended infrequent_stimulus shift_attention \n",
"15 attended infrequent_stimulus shift_attention \n",
"16 unattended infrequent_stimulus shift_attention \n",
"17 unattended infrequent_stimulus shift_attention \n",
"18 attended frequent_stimulus shift_attention \n",
"19 attended frequent_stimulus shift_attention \n",
"20 unattended frequent_stimulus shift_attention \n",
"21 unattended frequent_stimulus shift_attention \n",
"22 nan target_detected attend_auditory \n",
"23 nan target_detected attend_visual \n",
"24 nan target_detected shift_attention \n",
"25 nan target_detected shift_attention \n",
"26 nan nan attend_auditory \n",
"27 nan nan attend_visual \n",
"28 nan nan shift_attention \n",
"29 nan nan shift_attention \n",
"30 nan nan shift_attention \n"
"sub-002_task-FaceRecognition_events.tsv\n",
"sub-003_task-FaceRecognition_events.tsv\n",
"sub-004_task-FaceRecognition_events.tsv\n",
"sub-005_task-FaceRecognition_events.tsv\n",
"sub-006_task-FaceRecognition_events.tsv\n",
"sub-007_task-FaceRecognition_events.tsv\n",
"sub-008_task-FaceRecognition_events.tsv\n",
"sub-009_task-FaceRecognition_events.tsv\n",
"sub-010_task-FaceRecognition_events.tsv\n",
"sub-011_task-FaceRecognition_events.tsv\n",
"sub-012_task-FaceRecognition_events.tsv\n",
"sub-013_task-FaceRecognition_events.tsv\n",
"sub-014_task-FaceRecognition_events.tsv\n",
"sub-015_task-FaceRecognition_events.tsv\n",
"sub-016_task-FaceRecognition_events.tsv\n",
"sub-017_task-FaceRecognition_events.tsv\n",
"sub-018_task-FaceRecognition_events.tsv\n",
"sub-019_task-FaceRecognition_events.tsv\n",
"The total count of the keys is:31448\n",
" key_counts trial_type value\n",
"0 90 boundary 0\n",
"1 2700 famous_new 5\n",
"2 1313 famous_second_early 6\n",
"3 1291 famous_second_late 7\n",
"4 3532 left_nonsym 256\n",
"5 3381 left_sym 256\n",
"6 3616 right_nonsym 4096\n",
"7 4900 right_sym 4096\n",
"8 2700 scrambled_new 17\n",
"9 1271 scrambled_second_early 18\n",
"10 1334 scrambled_second_late 19\n",
"11 2700 unfamiliar_new 13\n",
"12 1304 unfamiliar_second_early 14\n",
"13 1316 unfamiliar_second_late 15\n"
]
}
],
Expand All @@ -114,22 +83,25 @@
"from hed.tools.util.io_util import get_file_list\n",
"\n",
"# Variables to set for the specific dataset\n",
"dataset_root_path = os.path.realpath('../../../datasets/eeg_ds002893s_hed_attention_shift')\n",
"data_root = 'T:/summaryTests/ds002718-download'\n",
"output_path = ''\n",
"exclude_dirs = ['stimuli']\n",
"exclude_dirs = ['stimuli', 'derivatives', 'code', 'sourcecode']\n",
"\n",
"# Construct the key map\n",
"key_columns = [ \"event_code\", \"cond_code\", \"event_type\", \"focus_modality\", \"attention_status\", \"task_role\", \"condition\"]\n",
"key_columns = [ \"trial_type\", \"value\"]\n",
"key_map = KeyMap(key_columns)\n",
"\n",
"# Construct the unique combinations\n",
"event_files = get_file_list(dataset_root_path, extensions=[\".tsv\"], name_suffix=\"_events\", exclude_dirs=exclude_dirs)\n",
"event_files = get_file_list(data_root, extensions=[\".tsv\"], name_suffix=\"_events\", exclude_dirs=exclude_dirs)\n",
"for event_file in event_files:\n",
" print(f\"{os.path.basename(event_file)}\")\n",
" df = get_new_dataframe(event_file)\n",
" key_map.update(df)\n",
"\n",
"key_map.resort()\n",
"template = key_map.make_template()\n",
"key_counts_sum = template['key_counts'].sum()\n",
"print(f\"The total count of the keys is:{key_counts_sum}\")\n",
"if output_path:\n",
" template.to_csv(output_path, sep='\\t', index=False, header=True)\n",
"else:\n",
Expand All @@ -138,8 +110,8 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
"end_time": "2023-09-03T19:43:55.968067600Z",
"start_time": "2023-09-03T19:43:48.336417200Z"
"end_time": "2023-10-24T20:08:40.958637400Z",
"start_time": "2023-10-24T20:08:24.603887900Z"
}
}
}
Expand Down
28 changes: 20 additions & 8 deletions hedcode/jupyter_notebooks/bids/validate_bids_dataset.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,18 @@
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"execution_count": 6,
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using HEDTOOLS version: {'date': '2023-10-12T10:29:43-0500', 'dirty': True, 'error': None, 'full-revisionid': 'e9e63e39f6a99487c5e69ae96e653c4580045ec0', 'version': '0.3.0+164.ge9e63e3.dirty'}\n",
"HED Examples version: {'version': '0.2.0+173.g63d5080.dirty', 'full-revisionid': '63d50808c4c7e97bdfc89076cb19af9e13252057', 'dirty': True, 'error': None, 'date': '2023-10-23T11:09:31-0500'}\n",
"No HED validation errors\n"
]
}
],
"source": [
"from hed.errors import get_printable_issue_string\n",
"from hed.tools import BidsDataset\n",
Expand All @@ -45,10 +55,8 @@
"\n",
"## Set the dataset location and the check_for_warnings flag\n",
"check_for_warnings = False\n",
"#dataset_path = '../../../datasets/eeg_ds003645s_hed_column'\n",
"#dataset_path = '../../../datasets/eeg_ds004105s_hed'\n",
"dataset_path = 't:/summaryTests/ds004105-download'\n",
"outfile = 't:/ds004105_errors.txt'\n",
"dataset_path = 't:/summaryTests/ds002718-download'\n",
"outfile = 't:/ds002718_errors.txt'\n",
"## Validate the dataset\n",
"bids = BidsDataset(dataset_path)\n",
"issue_list = bids.validate(check_for_warnings=check_for_warnings)\n",
Expand All @@ -57,12 +65,16 @@
"else:\n",
" issue_str = \"No HED validation errors\"\n",
"print(issue_str)\n",
"if outfile:\n",
"if outfile and issue_list:\n",
" with open(outfile, 'w') as fp:\n",
" fp.write(issue_str)\n"
],
"metadata": {
"collapsed": false
"collapsed": false,
"ExecuteTime": {
"end_time": "2023-10-24T18:51:33.095713100Z",
"start_time": "2023-10-24T18:51:31.396158Z"
}
}
}
],
Expand Down
14 changes: 7 additions & 7 deletions hedcode/jupyter_notebooks/remodeling/run_remodel_restore.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@
"This notebook uses `run_remodel_restore(arglist)` to restore the event files from\n",
"a backup created by `run_remodel_backup`.\n",
"\n",
"| parameter | default | dest | other | meaning |\n",
"| --------- | ------- | ---- | ----- | ------- |\n",
"| data_dir | | data_dir | | Full path of dataset root directory. |\n",
"| -n, --backup_name | default_back | backup_name | | Name of the default backup for remodeling. |\n",
"| -v, --verbose| | verbose | action=store_true | If present, output informative messages as computation progresses. |"
"| parameter | default | dest | other | meaning |\n",
"|--------------------| ------- | ---- | ----- | ------- |\n",
"| data_dir | | data_dir | | Full path of dataset root directory. |\n",
"| -bn, --backup_name | default_back | backup_name | | Name of the default backup for remodeling. |\n",
"| -v, --verbose | | verbose | action=store_true | If present, output informative messages as computation progresses. |"
]
},
{
Expand All @@ -31,7 +31,7 @@
"\n",
"backup_name = 'test_backup'\n",
"data_dir = os.path.realpath('../../../datasets/eeg_ds003645s_hed_remodel')\n",
"arg_list = [data_dir, '-n', backup_name]\n",
"arg_list = [data_dir, '-bn', backup_name]\n",
"cli_restore.main(arg_list)"
],
"metadata": {
Expand Down Expand Up @@ -63,4 +63,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
}
}
8 changes: 8 additions & 0 deletions hedcode/matlab_scripts/data_cleaning/getEventTable.m
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,13 @@
function eventTable = getEventTable(eventsFile, columnTypes, renameColumns)
% Read the table of events from the events file
%
% Parameters:
% eventsFile - the path of a BIDS tabular events file.
% columnTypes - a cell array of two-element values:
% column-name, column-type
% renameColumns - a cell array of two-element values:
% old-name, new-name
%
typeMap = containers.Map(columnTypes(:, 1), columnTypes(:, 2));
optsDect = detectImportOptions(eventsFile, 'FileType', 'delimitedtext');

Expand Down
45 changes: 2 additions & 43 deletions hedcode/matlab_scripts/data_cleaning/runEeglabImportEvents.m
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,49 +1,8 @@
%% This imports the _events.tsv into the corresponding EEG.set file

%% Set up the specifics for your dataset
filename = 'T:SummaryTests/ds004105-download/sub-01/ses-01/eeg/sub-01_ses-01_task-DriveRandomSound_run-1_events.tsv';
eventstruct = importevent( filename, [], 1024, 'key1', 'value1', ...);
% rootPath = '/XXX/SternbergWorkingPhaseTwo';
% setname = '';
% log_name = 'sternberg_12_import_events_log.txt';
% renameColumns = {'event_type', 'type'; 'onset', 'latency'};
%
% % rootPath = 'G:/AttentionShift/AttentionShiftWorkingPhaseTwo';
% % setname = 'Auditory Visual Attention Shift';
% % log_name = 'attention_shift_18_import_events_log.txt';
%
% % rootPath = 's:/bcit/AdvancedGuardDutyWorkingPhaseTwo';
% % setname = 'BCIT Advanced Guard Duty';
% % log_name = 'bcit_advanced_guard_duty_10_import_events_log.txt';
% %
% % rootPath = 's:/bcit/AuditoryCueingWorkingPhaseTwo';
% % setname = 'BCIT Auditory Cueing';
% % log_name = 'bcit_auditory_cueing_10_import_events_log.txt';
%
% % rootPath = 's:/bcit/BaselineDrivingWorkingPhaseTwo';
% % setname = 'BCIT Baseline Driving';
% % log_name = 'bcit_baseline_driving_10_import_events_log.txt';
%
% % rootPath = 's:/bcit/BasicGuardDutyWorkingPhaseTwo';
% % setname = 'BCIT Basic Guard Duty';
% % log_name = 'bcit_basic_guard_duty_10_import_events_log.txt';
%
% % rootPath = 's:/bcit/CalibrationDrivingWorkingPhaseTwo';
% % setname = 'BCIT Calibration Driving';
% % log_name = 'bcit_calibration_driving_10_import_events_log.txt';
%
% % rootPath = 's:/bcit/MindWanderingWorkingPhaseTwo';
% % setname = 'BCIT Mind Wandering';
% % log_name = 'bcit_mind_wandering_10_import_events_log.txt';
%
% % rootPath = 's:/bcit/SpeedControlWorkingPhaseTwo';
% % setname = 'BCIT Speed Control';
% % log_name = 'bcit_speed_control_10_import_events_log.txt';
%
% % rootPath = 's:/bcit/TrafficComplexityWorkingPhaseTwo';
% % setname = 'BCIT Traffic Complexity';
% % log_name = 'bcit_traffic_complexity_10_import_events_log.txt';
%
filename = 'T:/SummaryTests/ds004105-download/sub-01/ses-01/eeg/sub-01_ses-01_task-DriveRandomSound_run-1_events.tsv';

% excludeDirs = {'sourcedata', 'code', 'stimuli', 'derivatives'};
% namePrefix = '';
% nameSuffix = '_eeg';
Expand Down
6 changes: 3 additions & 3 deletions hedcode/matlab_scripts/web_services/runAllTests.m
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
dochost = 'https://hedtools.org/hed';
%host = 'https://hedtools.org/hed_dev';
host = 'http://127.0.0.1:5000';
host = 'https://hedtools.org/hed';
host = 'https://hedtools.org/hed_dev';
%host = 'http://127.0.0.1:5000';


errorMap = containers.Map('KeyType', 'char', 'ValueType', 'any');
Expand Down
32 changes: 16 additions & 16 deletions hedcode/matlab_scripts/web_services/testEventSearchServices.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
'schema_version', '8.0.0', ...
'sidecar_string', data.jsonText, ...
'events_string', data.eventsText, ...
'query', '[[Intended-effect, Cue]]');
'query', '{Intended-effect, Cue}');

response1 = webwrite(servicesUrl, request1, options);
response1 = jsondecode(response1);
Expand All @@ -25,19 +25,19 @@
errors{end + 1} = 'Example 1 failed execute the search.';
end

% %% Example 2: Search an events file for HED
% request2 = struct('service', 'events_search', ...
% 'schema_version', '8.0.0', ...
% 'sidecar_string', data.jsonText, ...
% 'events_string', data.eventsText, ...
% 'columns_included', '', ...
% 'query', '[[Intended-effect, Cue]]');
% request2.columns_included = {'onset'};
% response2 = webwrite(servicesUrl, request2, options);
% response2 = jsondecode(response2);
% outputReport(response2, 'Example 2 Querying an events file with extra columns');
% if ~isempty(response2.error_type) || ...
% ~strcmpi(response2.results.msg_category, 'success')
% errors{end + 1} = 'Example 2 failed execute the search.';
% end
%% Example 2: Search an events file for HED
request2 = struct('service', 'events_search', ...
'schema_version', '8.0.0', ...
'sidecar_string', data.jsonText, ...
'events_string', data.eventsText, ...
'columns_included', '', ...
'query', '{Intended-effect, Cue}');
request2.columns_included = {'onset'};
response2 = webwrite(servicesUrl, request2, options);
response2 = jsondecode(response2);
outputReport(response2, 'Example 2 Querying an events file with extra columns');
if ~isempty(response2.error_type) || ...
~strcmpi(response2.results.msg_category, 'success')
errors{end + 1} = 'Example 2 failed execute the search.';
end