From e1dd99e9da9088418f2904a144c183f826008726 Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Wed, 20 Mar 2024 15:23:25 +0100 Subject: [PATCH 1/8] account for optional Z in time strs --- mne_bids/read.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mne_bids/read.py b/mne_bids/read.py index 365cdd220..0836989e5 100644 --- a/mne_bids/read.py +++ b/mne_bids/read.py @@ -331,6 +331,8 @@ def _handle_scans_reading(scans_fname, raw, bids_path): if "." not in acq_time: # acquisition time ends with '.%fZ' microseconds string acq_time += ".0Z" + if "Z" not in acq_time: + acq_time += "Z" acq_time = datetime.strptime(acq_time, "%Y-%m-%dT%H:%M:%S.%fZ") acq_time = acq_time.replace(tzinfo=timezone.utc) From 563af7b4ee2d977b4e8b0b71386f4d182a30f6b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Mon, 25 Mar 2024 13:27:46 +0100 Subject: [PATCH 2/8] Try to handle UTC zero offset and local time zones correctly --- mne_bids/read.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/mne_bids/read.py b/mne_bids/read.py index 0836989e5..41e34bff7 100644 --- a/mne_bids/read.py +++ b/mne_bids/read.py @@ -327,14 +327,32 @@ def _handle_scans_reading(scans_fname, raw, bids_path): # extract the acquisition time from scans file acq_time = acq_times[row_ind] if acq_time != "n/a": - # microseconds in the acquisition time is optional + # BIDS allows the time to be stored in UTC with a time-zone offset, which is + # indicated by a trailing "Z" in the datetime string. If the "Z" is missing, the + # time is represented as "local" time. We have no way to know what the local + # time zone is at the acquisition site; so we simply assume the same time zone + # as the user's current system (this is what spec demands anyway). + acq_time_is_utc = acq_time.endswith("Z") + + # microseconds part in the acquisition time is optional; add it if missing if "." not in acq_time: - # acquisition time ends with '.%fZ' microseconds string - acq_time += ".0Z" - if "Z" not in acq_time: - acq_time += "Z" - acq_time = datetime.strptime(acq_time, "%Y-%m-%dT%H:%M:%S.%fZ") - acq_time = acq_time.replace(tzinfo=timezone.utc) + if acq_time_is_utc: + acq_time = acq_time.replace("Z", ".0Z") + else: + acq_time += ".0" + + dt_string = "%Y-%m-%dT%H:%M:%S.%fZ" + if acq_time_is_utc: + dt_string += "Z" + + acq_time = datetime.strptime(acq_time, dt_string) + + if acq_time_is_utc: + # Enforce setting timezone to UTC without additonal conversion + acq_time = acq_time.replace(tzinfo=timezone.utc) + else: + # Convert time offset to UTC + acq_time = acq_time.astimezone(timezone.utc) logger.debug( f"Loaded {scans_fname} scans file to set " f"acq_time as {acq_time}." From d02c8ef091ca15c1bdc17e17f2c9f930d9dce39f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Mon, 25 Mar 2024 13:31:24 +0100 Subject: [PATCH 3/8] Fix comment --- mne_bids/read.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mne_bids/read.py b/mne_bids/read.py index 41e34bff7..a769923ca 100644 --- a/mne_bids/read.py +++ b/mne_bids/read.py @@ -327,10 +327,10 @@ def _handle_scans_reading(scans_fname, raw, bids_path): # extract the acquisition time from scans file acq_time = acq_times[row_ind] if acq_time != "n/a": - # BIDS allows the time to be stored in UTC with a time-zone offset, which is + # BIDS allows the time to be stored in UTC with a zero time-zone offset, as # indicated by a trailing "Z" in the datetime string. If the "Z" is missing, the # time is represented as "local" time. We have no way to know what the local - # time zone is at the acquisition site; so we simply assume the same time zone + # time zone is at the *acquisition* site; so we simply assume the same time zone # as the user's current system (this is what spec demands anyway). acq_time_is_utc = acq_time.endswith("Z") From 34f8490394cfc4346d2b991d8cc984d7e4771594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Mon, 25 Mar 2024 13:33:31 +0100 Subject: [PATCH 4/8] Fix --- mne_bids/read.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne_bids/read.py b/mne_bids/read.py index a769923ca..d77cf8358 100644 --- a/mne_bids/read.py +++ b/mne_bids/read.py @@ -341,7 +341,7 @@ def _handle_scans_reading(scans_fname, raw, bids_path): else: acq_time += ".0" - dt_string = "%Y-%m-%dT%H:%M:%S.%fZ" + dt_string = "%Y-%m-%dT%H:%M:%S.%f" if acq_time_is_utc: dt_string += "Z" From 5ffa7c54d15755c4464efc3d852d717e9f826a45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Mon, 25 Mar 2024 13:37:48 +0100 Subject: [PATCH 5/8] Better variable name --- mne_bids/read.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mne_bids/read.py b/mne_bids/read.py index d77cf8358..2204449b4 100644 --- a/mne_bids/read.py +++ b/mne_bids/read.py @@ -341,11 +341,11 @@ def _handle_scans_reading(scans_fname, raw, bids_path): else: acq_time += ".0" - dt_string = "%Y-%m-%dT%H:%M:%S.%f" + date_format = "%Y-%m-%dT%H:%M:%S.%f" if acq_time_is_utc: - dt_string += "Z" + date_format += "Z" - acq_time = datetime.strptime(acq_time, dt_string) + acq_time = datetime.strptime(acq_time, date_format) if acq_time_is_utc: # Enforce setting timezone to UTC without additonal conversion From 164dd4f1da02879afad54863f121804a62c2c3c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Mon, 25 Mar 2024 14:05:48 +0100 Subject: [PATCH 6/8] Add tests --- mne_bids/tests/test_read.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/mne_bids/tests/test_read.py b/mne_bids/tests/test_read.py index ca83298ad..7d489ca35 100644 --- a/mne_bids/tests/test_read.py +++ b/mne_bids/tests/test_read.py @@ -601,7 +601,7 @@ def test_handle_scans_reading(tmp_path): acq_time_str = scans_tsv["acq_time"][0] acq_time = datetime.strptime(acq_time_str, "%Y-%m-%dT%H:%M:%S.%fZ") acq_time = acq_time.replace(tzinfo=timezone.utc) - new_acq_time = acq_time_str.split(".")[0] + new_acq_time = acq_time_str.split(".")[0] + "Z" assert acq_time == raw_01.info["meas_date"] scans_tsv["acq_time"][0] = new_acq_time _to_tsv(scans_tsv, scans_path) @@ -609,12 +609,30 @@ def test_handle_scans_reading(tmp_path): # now re-load the data and it should be different # from the original date and the same as the newly altered date raw_02 = read_raw_bids(bids_path) - new_acq_time += ".0Z" + new_acq_time = new_acq_time.replace("Z", ".0Z") new_acq_time = datetime.strptime(new_acq_time, "%Y-%m-%dT%H:%M:%S.%fZ") new_acq_time = new_acq_time.replace(tzinfo=timezone.utc) assert raw_02.info["meas_date"] == new_acq_time assert new_acq_time != raw_01.info["meas_date"] + # Test without optional zero-offset UTC time-zone indicator (i.e., without trailing + # "Z") + for has_microsecs in (True, False): + new_acq_time_str = "2002-12-03T19:01:10" + date_format = "%Y-%m-%dT%H:%M:%S" + if has_microsecs: + new_acq_time_str += ".0" + date_format += ".%f" + + scans_tsv["acq_time"][0] = new_acq_time_str + _to_tsv(scans_tsv, scans_path) + + # now re-load the data and it should be different + # from the original date and the same as the newly altered date + raw_03 = read_raw_bids(bids_path) + new_acq_time = datetime.strptime(new_acq_time_str, date_format) + assert raw_03.info["meas_date"] == new_acq_time.astimezone(timezone.utc) + @pytest.mark.filterwarnings(warning_str["channel_unit_changed"]) def test_handle_scans_reading_brainvision(tmp_path): From 856678407c68e558980f2f2eab4b63e333c3a6c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Mon, 25 Mar 2024 14:12:17 +0100 Subject: [PATCH 7/8] Update mne_bids/read.py --- mne_bids/read.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne_bids/read.py b/mne_bids/read.py index 2204449b4..375b427ce 100644 --- a/mne_bids/read.py +++ b/mne_bids/read.py @@ -331,7 +331,7 @@ def _handle_scans_reading(scans_fname, raw, bids_path): # indicated by a trailing "Z" in the datetime string. If the "Z" is missing, the # time is represented as "local" time. We have no way to know what the local # time zone is at the *acquisition* site; so we simply assume the same time zone - # as the user's current system (this is what spec demands anyway). + # as the user's current system (this is what the spec demands anyway). acq_time_is_utc = acq_time.endswith("Z") # microseconds part in the acquisition time is optional; add it if missing From 92273120cd673de6ca9ce06b0643ba4d1db28745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Richard=20H=C3=B6chenberger?= Date: Mon, 25 Mar 2024 14:55:43 +0100 Subject: [PATCH 8/8] Add changelog entry --- doc/whats_new.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/whats_new.rst b/doc/whats_new.rst index 0a13f0311..66eea1fc8 100644 --- a/doc/whats_new.rst +++ b/doc/whats_new.rst @@ -62,6 +62,8 @@ Detailed list of changes ``pandas.Int64Dtype`` instead of ``float64``, by `Eric Larson`_ (:gh:`1227`) - The :func:`mne_bids.copyfiles.copyfile_ctf` now accounts for files with ``.{integer}_meg4`` extension, instead of only .meg4, when renaming the files of a .ds folder, by `Mara Wolter`_ (:gh:`1230`) +- We fixed handling of time zones when reading ``*_scans.tsv`` files; specifically, non-UTC timestamps are now processed correctly, + by `Stefan Appelhoff`_ and `Richard Höchenberger`_ (:gh:`1240`) ⚕️ Code health ^^^^^^^^^^^^^^