From 7792c603fe16d3466270c0c3b1fb11f7eb38d694 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Sat, 28 Oct 2023 15:15:41 +0200 Subject: [PATCH 1/8] fix timestamps fictrac --- .../behavior/fictrac/fictracdatainterface.py | 95 +++++++++++++++---- .../test_on_data/test_behavior_interfaces.py | 4 +- 2 files changed, 81 insertions(+), 18 deletions(-) diff --git a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py index 9ec9bd978..5c9770312 100644 --- a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py @@ -1,4 +1,5 @@ -import importlib.util +from __future__ import annotations + from datetime import datetime, timezone from pathlib import Path from typing import Optional @@ -172,7 +173,8 @@ def get_metadata(self): metadata = super().get_metadata() session_start_time = extract_session_start_time(self.file_path) - metadata["NWBFile"].update(session_start_time=session_start_time) + if session_start_time: + metadata["NWBFile"].update(session_start_time=session_start_time) return metadata @@ -193,7 +195,7 @@ def add_to_nwbfile( import pandas as pd # The first row only contains the session start time and invalid data - fictrac_data_df = pd.read_csv(self.file_path, sep=",", skiprows=1, header=None, names=self.columns_in_dat_file) + fictrac_data_df = pd.read_csv(self.file_path, sep=",", skiprows=0, header=None, names=self.columns_in_dat_file) # Get the timestamps timestamps = self.get_timestamps() @@ -243,19 +245,50 @@ def add_to_nwbfile( processing_module.add_data_interface(position_container) def get_original_timestamps(self): + """ + In FicTrac version 2.1.1 the timestampts that are 0 are changed for system time in milliseconds. The + system time is set at the point in which the frame is processed. This means that the timestamps are not + monotonically increasing. This function checks if the timestamps are inconsistent. + + This happens for video sources whe OpenCV returns 0 at the beginning: + [0, t1, t2, t3, ...] + + In that case, FicTrac changes the 0 to system time: + + [system_time, t1, t2, t3, ...] + + And the timestamps are inconsistent + + See: + https://github.com/rjdmoore/fictrac/issues/29 + + """ + import pandas as pd timestamp_index = self.columns_in_dat_file.index("timestamp") - fictrac_data_df = pd.read_csv(self.file_path, sep=",", skiprows=1, header=None, usecols=[timestamp_index]) + fictrac_data_df = pd.read_csv(self.file_path, sep=",", skiprows=0, header=None, usecols=[timestamp_index]) + + timestamps = fictrac_data_df[timestamp_index].values / 1000.0 # Transform to seconds - return fictrac_data_df[timestamp_index].values / 1000.0 + # Correct for the case when 0 timestamp was replaced by system time + first_difference = timestamps[1] - timestamps[0] + if first_difference < 0: + timestamps[0] = 0.0 + + first_timestamp = timestamps[0] + + # Heuristic to test if timestamps are in Unix epoch + length_in_seconds_of_a_10_year_experiment = 10 * 365 * 24 * 60 * 60 + if first_timestamp > length_in_seconds_of_a_10_year_experiment: + timestamps = timestamps - timestamps[0] + + return timestamps def get_timestamps(self): timestamps = self._timestamps if self._timestamps is not None else self.get_original_timestamps() if self._starting_time is not None: - # Shift the timestamps to the starting time such that timestamps[0] == self._starting_time - # timestamps = timestamps - timestamps[0] + self._starting_time timestamps = timestamps + self._starting_time return timestamps @@ -267,24 +300,54 @@ def set_aligned_starting_time(self, aligned_starting_time): self._starting_time = aligned_starting_time -def extract_session_start_time(file_path: FilePathType) -> datetime: +def extract_session_start_time( + file_path: FilePathType, + configuration_file_path: Optional[FilePathType] = None, +) -> datetime | None: """ - Lazily extract the session start datetime from a FicTrac data file. + Extract the session start time from FicTrac data or configuration file. - In FicTrac the column 22 in the data has the timestamps which are given in milliseconds since the epoch. + Parameters + ---------- + file_path : Path + Path to the FicTrac data file. + configuration_file_path : Optional[Path] + Path to the FicTrac configuration file. If not provided, the function will look for "fictrac_config.txt" + in the same directory as the data file. + + Returns + ------- + datetime | None + The session start time of in UTC as a datetime object. `None` if the session start time cannot be extracted. + + Notes + ----- + - In FicTrac, column 22 of the data file contains timestamps in milliseconds. + - If the timestamp is since the Unix epoch (1970-01-01 00:00:00 UTC), it's transformed to UTC. + - If timestamps are since the start of the session, the session start time is extracted from the configuration file. + - If neither applies or the relevant files are not found or correctly formatted, returns None. - The epoch in Linux is 1970-01-01 00:00:00 UTC. """ with open(file_path, "r") as file: - # Read the first data line first_line = file.readline() - # Split by comma and extract the timestamp (the 22nd column) - utc_timestamp = float(first_line.split(",")[21]) / 1000.0 # Transform to seconds + first_timestamp = float(first_line.split(",")[21]) / 1000.0 # Convert to seconds + + # Heuristic to test if timestamps are in Unix epoch + length_in_seconds_of_a_10_year_experiment = 10 * 365 * 24 * 60 * 60 + if first_timestamp > length_in_seconds_of_a_10_year_experiment: + utc_timestamp = first_timestamp + return datetime.utcfromtimestamp(utc_timestamp).replace(tzinfo=timezone.utc) - utc_datetime = datetime.utcfromtimestamp(utc_timestamp).replace(tzinfo=timezone.utc) + if configuration_file_path is None: + configuration_file_path = file_path.parent / "fictrac_config.txt" + if configuration_file_path.is_file(): + configuration_file = parse_fictrac_config(configuration_file_path) + session_start_time = datetime.strptime(configuration_file.get("build_date", ""), "%b %d %Y") + return session_start_time.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=timezone.utc) - return utc_datetime + # If neither of the the methods above work, return None + return None # TODO: Parse probably will do this in a simpler way. diff --git a/tests/test_on_data/test_behavior_interfaces.py b/tests/test_on_data/test_behavior_interfaces.py index 69c1d748b..1f1ebfb6a 100644 --- a/tests/test_on_data/test_behavior_interfaces.py +++ b/tests/test_on_data/test_behavior_interfaces.py @@ -47,7 +47,7 @@ def check_read_nwb(self, nwbfile_path: str): # This is currently structured to with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: nwbfile = io.read() - fictrac_position_container = nwbfile.processing["Behavior"].data_interfaces["FicTrac"] + fictrac_position_container = nwbfile.processing["behavior"].data_interfaces["FicTrac"] assert isinstance(fictrac_position_container, Position) assert len(fictrac_position_container.spatial_series) == 10 @@ -82,7 +82,7 @@ def check_read_nwb(self, nwbfile_path: str): # This is currently structured to with NWBHDF5IO(path=nwbfile_path, mode="r", load_namespaces=True) as io: nwbfile = io.read() - fictrac_position_container = nwbfile.processing["Behavior"].data_interfaces["FicTrac"] + fictrac_position_container = nwbfile.processing["behavior"].data_interfaces["FicTrac"] assert isinstance(fictrac_position_container, Position) assert len(fictrac_position_container.spatial_series) == 10 From eb66dfc4aee5ef361d8d12f09a607db499ae3946 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Sat, 28 Oct 2023 15:24:51 +0200 Subject: [PATCH 2/8] docstring --- .../behavior/fictrac/fictracdatainterface.py | 43 ++++++++++++------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py index 5c9770312..be55b56bc 100644 --- a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py @@ -246,20 +246,34 @@ def add_to_nwbfile( def get_original_timestamps(self): """ - In FicTrac version 2.1.1 the timestampts that are 0 are changed for system time in milliseconds. The - system time is set at the point in which the frame is processed. This means that the timestamps are not - monotonically increasing. This function checks if the timestamps are inconsistent. - - This happens for video sources whe OpenCV returns 0 at the beginning: - [0, t1, t2, t3, ...] - - In that case, FicTrac changes the 0 to system time: - - [system_time, t1, t2, t3, ...] - - And the timestamps are inconsistent - - See: + Retrieve and correct the original timestamps from a FicTrac data file. + + This function performs two key corrections: + 1. If the first timestamp is replaced with system time (e.g., in the case of the first timestamp being 0), + it is reset back to 0. This issue has been identified in FicTrac version 2.1.1 and possibly in earlier versions. + + 2. If timestamps are recorded in Unix epoch time format, indicating an absolute time reference since + the Unix epoch (1970-01-01 00:00:00 UTC), the entire series is re-centered by + subtracting the first timestamp value. + + Notes + ----- + - The case of the first timestamp being replaced with 0 is prominent in data whose source is a video file. + In that case, if OpenCV returns the first timestamps as 0, FicTrac replaces it with the system time. + This action results in an inconsistent sequence like [system_time, t1, t2, t3, ...]. + The function adjusts this by changing the first timestamp back to 0. + - The function also detects timestamps in Unix epoch format, which might be uncharacteristically large numbers, + representing the time elapsed since the Unix epoch. Such timestamps are re-centered to the start of the experiment + for consistency. + + Returns + ------- + np.ndarray + An array of corrected timestamps, in seconds. + + References + ---------- + Issue discussion on FicTrac's timestamp inconsistencies: https://github.com/rjdmoore/fictrac/issues/29 """ @@ -267,7 +281,6 @@ def get_original_timestamps(self): import pandas as pd timestamp_index = self.columns_in_dat_file.index("timestamp") - fictrac_data_df = pd.read_csv(self.file_path, sep=",", skiprows=0, header=None, usecols=[timestamp_index]) timestamps = fictrac_data_df[timestamp_index].values / 1000.0 # Transform to seconds From 05f232510f165f7e2c172e92060fff6f3e593d68 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Sat, 28 Oct 2023 15:42:11 +0200 Subject: [PATCH 3/8] add test --- .../behavior/fictrac/fictracdatainterface.py | 16 ++++++---------- tests/test_on_data/test_behavior_interfaces.py | 4 ++++ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py index be55b56bc..5f8895162 100644 --- a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py @@ -1,8 +1,6 @@ -from __future__ import annotations - from datetime import datetime, timezone from pathlib import Path -from typing import Optional +from typing import Optional, Union from pynwb.behavior import Position, SpatialSeries from pynwb.file import NWBFile @@ -195,11 +193,10 @@ def add_to_nwbfile( import pandas as pd # The first row only contains the session start time and invalid data - fictrac_data_df = pd.read_csv(self.file_path, sep=",", skiprows=0, header=None, names=self.columns_in_dat_file) + fictrac_data_df = pd.read_csv(self.file_path, sep=",", names=self.columns_in_dat_file) # Get the timestamps timestamps = self.get_timestamps() - starting_time = timestamps[0] # Note: The last values of the timestamps look very irregular for the sample file in catalyst neuro gin repo @@ -281,18 +278,17 @@ def get_original_timestamps(self): import pandas as pd timestamp_index = self.columns_in_dat_file.index("timestamp") - fictrac_data_df = pd.read_csv(self.file_path, sep=",", skiprows=0, header=None, usecols=[timestamp_index]) + fictrac_data_df = pd.read_csv(self.file_path, sep=",", usecols=[timestamp_index]) timestamps = fictrac_data_df[timestamp_index].values / 1000.0 # Transform to seconds - # Correct for the case when 0 timestamp was replaced by system time + # Correct for the case when the first timestamp was replaced by system time first_difference = timestamps[1] - timestamps[0] if first_difference < 0: timestamps[0] = 0.0 - first_timestamp = timestamps[0] - # Heuristic to test if timestamps are in Unix epoch + first_timestamp = timestamps[0] length_in_seconds_of_a_10_year_experiment = 10 * 365 * 24 * 60 * 60 if first_timestamp > length_in_seconds_of_a_10_year_experiment: timestamps = timestamps - timestamps[0] @@ -316,7 +312,7 @@ def set_aligned_starting_time(self, aligned_starting_time): def extract_session_start_time( file_path: FilePathType, configuration_file_path: Optional[FilePathType] = None, -) -> datetime | None: +) -> Union[datetime, None]: """ Extract the session start time from FicTrac data or configuration file. diff --git a/tests/test_on_data/test_behavior_interfaces.py b/tests/test_on_data/test_behavior_interfaces.py index 1f1ebfb6a..a465dd925 100644 --- a/tests/test_on_data/test_behavior_interfaces.py +++ b/tests/test_on_data/test_behavior_interfaces.py @@ -65,6 +65,8 @@ def check_read_nwb(self, nwbfile_path: str): # This is currently structured to assert spatial_series.unit == expected_units assert spatial_series.conversion == 1.0 + assert spatial_series.timestamps[0] == 0.0 + class TestFicTracDataInterfaceWithRadius(DataInterfaceTestMixin, unittest.TestCase): data_interface_cls = FicTracDataInterface @@ -99,6 +101,8 @@ def check_read_nwb(self, nwbfile_path: str): # This is currently structured to assert spatial_series.unit == expected_units assert spatial_series.conversion == self.interface.radius + assert spatial_series.timestamps[0] == 0.0 + class TestFicTracDataInterfaceTiming(TemporalAlignmentMixin, unittest.TestCase): data_interface_cls = FicTracDataInterface From b593fc1c469d03aaf2e668d607e472f4ca62f52f Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Sat, 28 Oct 2023 15:47:31 +0200 Subject: [PATCH 4/8] changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a544a7a9e..a9311577e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * Added alignment methods support to `MockRecordingInterface` [PR #611](https://github.com/catalystneuro/neuroconv/pull/611) * Added `NeuralynxNvtInterface`, which can read position tracking NVT files. [PR #580](https://github.com/catalystneuro/neuroconv/pull/580) * Adding radius as a conversion factor in `FicTracDataInterface`. [PR #619](https://github.com/catalystneuro/neuroconv/pull/619) +* Coerce `FicTracDataInterface` original timestamps to start from 0. [PR #619](https://github.com/catalystneuro/neuroconv/pull/619) ### Fixes * Remove `starting_time` reset to default value (0.0) when adding the rate and updating the `photon_series_kwargs` or `roi_response_series_kwargs`, in `add_photon_series` or `add_fluorescence_traces`. [PR #595](https://github.com/catalystneuro/neuroconv/pull/595) @@ -22,7 +23,7 @@ * Pin lower bound HDMF version to `3.10.0`. [PR #586](https://github.com/catalystneuro/neuroconv/pull/586) ### Deprecation -* Removed `use_times` and `buffer_size` from `add_photon_series`. [PR #600](https://github.com/catalystneuro/neuroconv/pull/600) +* Removed `use_times` and `buffer_size` from `add_photon_series`. [PR #621](https://github.com/catalystneuro/neuroconv/pull/621) ### Testing From 0b6d31a75fbcc5c11741722f9d2eca0b5611e11d Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Sat, 28 Oct 2023 15:53:25 +0200 Subject: [PATCH 5/8] fix mistake in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9311577e..ffe6cc14b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ * Pin lower bound HDMF version to `3.10.0`. [PR #586](https://github.com/catalystneuro/neuroconv/pull/586) ### Deprecation -* Removed `use_times` and `buffer_size` from `add_photon_series`. [PR #621](https://github.com/catalystneuro/neuroconv/pull/621) +* Removed `use_times` and `buffer_size` from `add_photon_series`. [PR #600](https://github.com/catalystneuro/neuroconv/pull/600) ### Testing From 44966f861160bbd7031d4aa1d0c35b8d27ea7933 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Sat, 28 Oct 2023 16:41:40 +0200 Subject: [PATCH 6/8] restore header None --- .../datainterfaces/behavior/fictrac/fictracdatainterface.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py index 5f8895162..7bf57b72e 100644 --- a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py @@ -193,7 +193,7 @@ def add_to_nwbfile( import pandas as pd # The first row only contains the session start time and invalid data - fictrac_data_df = pd.read_csv(self.file_path, sep=",", names=self.columns_in_dat_file) + fictrac_data_df = pd.read_csv(self.file_path, sep=",", header=None, names=self.columns_in_dat_file) # Get the timestamps timestamps = self.get_timestamps() @@ -278,7 +278,7 @@ def get_original_timestamps(self): import pandas as pd timestamp_index = self.columns_in_dat_file.index("timestamp") - fictrac_data_df = pd.read_csv(self.file_path, sep=",", usecols=[timestamp_index]) + fictrac_data_df = pd.read_csv(self.file_path, sep=",", header=None, usecols=[timestamp_index]) timestamps = fictrac_data_df[timestamp_index].values / 1000.0 # Transform to seconds From b6ec116096118e50a3f4079ea9e3887d0a441a81 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 31 Oct 2023 13:21:11 +0100 Subject: [PATCH 7/8] fictract interface --- .../behavior/fictrac/fictracdatainterface.py | 94 +++++++++++-------- 1 file changed, 55 insertions(+), 39 deletions(-) diff --git a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py index 7bf57b72e..a43f35190 100644 --- a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py @@ -2,6 +2,7 @@ from pathlib import Path from typing import Optional, Union +import numpy as np from pynwb.behavior import Position, SpatialSeries from pynwb.file import NWBFile @@ -21,6 +22,7 @@ class FicTracDataInterface(BaseTemporalAlignmentInterface): "visual fixation", ] + timestamps_column = 21 # Columns in the .dat binary file with the data. The full description of the header can be found in: # https://github.com/rjdmoore/fictrac/blob/master/doc/data_header.txt columns_in_dat_file = [ @@ -162,6 +164,7 @@ def __init__( self.verbose = verbose self._timestamps = None self.radius = radius + self.configuration_file_path = None super().__init__(file_path=file_path) self._timestamps = None @@ -170,7 +173,10 @@ def __init__( def get_metadata(self): metadata = super().get_metadata() - session_start_time = extract_session_start_time(self.file_path) + session_start_time = extract_session_start_time( + file_path=self.file_path, + configuration_file_path=self.configuration_file_path, + ) if session_start_time: metadata["NWBFile"].update(session_start_time=session_start_time) @@ -192,7 +198,6 @@ def add_to_nwbfile( import pandas as pd - # The first row only contains the session start time and invalid data fictrac_data_df = pd.read_csv(self.file_path, sep=",", header=None, names=self.columns_in_dat_file) # Get the timestamps @@ -243,55 +248,60 @@ def add_to_nwbfile( def get_original_timestamps(self): """ - Retrieve and correct the original timestamps from a FicTrac data file. + Retrieve and correct timestamps from a FicTrac data file. - This function performs two key corrections: - 1. If the first timestamp is replaced with system time (e.g., in the case of the first timestamp being 0), - it is reset back to 0. This issue has been identified in FicTrac version 2.1.1 and possibly in earlier versions. + This function addresses two specific issues with timestamps in FicTrac data: - 2. If timestamps are recorded in Unix epoch time format, indicating an absolute time reference since - the Unix epoch (1970-01-01 00:00:00 UTC), the entire series is re-centered by - subtracting the first timestamp value. + 1. Resetting Initial Timestamp: + In some instances, FicTrac replaces the initial timestamp (0) with the system time. This commonly occurs + when the data source is a video file, and OpenCV reports the first timestamp as 0. Since OpenCV also + uses 0 as a marker for invalid values, FicTrac defaults to system time in that case. This leads to + inconsistent timestamps like [system_time, t1, t2, t3, ...]. The function corrects this by resetting the + first timestamp back to 0 when a negative difference is detected between the first two timestamps. - Notes - ----- - - The case of the first timestamp being replaced with 0 is prominent in data whose source is a video file. - In that case, if OpenCV returns the first timestamps as 0, FicTrac replaces it with the system time. - This action results in an inconsistent sequence like [system_time, t1, t2, t3, ...]. - The function adjusts this by changing the first timestamp back to 0. - - The function also detects timestamps in Unix epoch format, which might be uncharacteristically large numbers, - representing the time elapsed since the Unix epoch. Such timestamps are re-centered to the start of the experiment - for consistency. + 2. Re-centering Unix Epoch Time: + If timestamps are in Unix epoch time format (time since 1970-01-01 00:00:00 UTC), this function re-centers + the time series by subtracting the first timestamp. This adjustment ensures that timestamps represent the + elapsed time since the start of the experiment rather than the Unix epoch. This case appears when one of the + sources of data in FicTrac (such as PGR or Basler) lacks a timestamp extraction method. FicTrac + then falls back to using the system time, which is in Unix epoch format. Returns ------- np.ndarray An array of corrected timestamps, in seconds. + Notes + ----- + - The issue of the initial timestamp replacement appears in FicTrac 2.1.1 and earlier versions. + - Re-centering is essential for timestamps in Unix epoch format as timestamps in an NWB file must be relative + to the start of the session. The heuristic here is to check if the first timestamp is larger than the length + of a 10-year experiment in seconds. If so, it's assumed that the timestamps are in Unix epoch format. + References ---------- Issue discussion on FicTrac's timestamp inconsistencies: https://github.com/rjdmoore/fictrac/issues/29 - """ import pandas as pd - timestamp_index = self.columns_in_dat_file.index("timestamp") - fictrac_data_df = pd.read_csv(self.file_path, sep=",", header=None, usecols=[timestamp_index]) + fictrac_data_df = pd.read_csv(self.file_path, sep=",", header=None, usecols=[self.timestamps_column]) - timestamps = fictrac_data_df[timestamp_index].values / 1000.0 # Transform to seconds + timestamps = fictrac_data_df[self.timestamps_column].values / 1000.0 # Transform to seconds - # Correct for the case when the first timestamp was replaced by system time + # Correct for the case when only the first timestamp was replaced by system time first_difference = timestamps[1] - timestamps[0] if first_difference < 0: timestamps[0] = 0.0 # Heuristic to test if timestamps are in Unix epoch - first_timestamp = timestamps[0] length_in_seconds_of_a_10_year_experiment = 10 * 365 * 24 * 60 * 60 - if first_timestamp > length_in_seconds_of_a_10_year_experiment: + all_timestamps_are_in_unix_epoch = np.all(timestamps > length_in_seconds_of_a_10_year_experiment) + if all_timestamps_are_in_unix_epoch: timestamps = timestamps - timestamps[0] + # TODO: If we agree to ALWAYS constrain timestamps to be relative to the start of the session, we can + # Always shift here and remove the heuristic above. return timestamps @@ -314,33 +324,39 @@ def extract_session_start_time( configuration_file_path: Optional[FilePathType] = None, ) -> Union[datetime, None]: """ - Extract the session start time from FicTrac data or configuration file. + Extract the session start time from a FicTrac data file or its configuration file. + + The session start time is determined from the data file if the timestamps are in Unix epoch format. If not, the + function defaults to extracting the date from the configuration file and assuming that the start time is midnight. + If neither of these methods works, the function returns None. + + The session start time, has two different meanings depending on the source of the FicTrac data: + - For video file sources (.avi, .mp4, etc), the session start time corresponds to the time when the + FicTrac analysis commenced. That is, the session start time reflects the analysis time rather than + the actual start of the experiment. + - For camera sources (such as PGR or Basler), the sesion start time is either the time reported by the camera + or the system time if the camera's SDK does not provide timestamps to Fictrac. In both cases, this time is + the experiment start time, barring synchronization issues. Parameters ---------- - file_path : Path + file_path : FilePathType Path to the FicTrac data file. - configuration_file_path : Optional[Path] - Path to the FicTrac configuration file. If not provided, the function will look for "fictrac_config.txt" - in the same directory as the data file. + configuration_file_path : Optional[FilePathType] + Path to the FicTrac configuration file. If omitted, the function defaults to searching for + "fictrac_config.txt" in the same directory as the data file. Returns ------- datetime | None The session start time of in UTC as a datetime object. `None` if the session start time cannot be extracted. - Notes - ----- - - In FicTrac, column 22 of the data file contains timestamps in milliseconds. - - If the timestamp is since the Unix epoch (1970-01-01 00:00:00 UTC), it's transformed to UTC. - - If timestamps are since the start of the session, the session start time is extracted from the configuration file. - - If neither applies or the relevant files are not found or correctly formatted, returns None. - """ with open(file_path, "r") as file: first_line = file.readline() - first_timestamp = float(first_line.split(",")[21]) / 1000.0 # Convert to seconds + timestamps_column = FicTracDataInterface.timestamps_column + first_timestamp = float(first_line.split(",")[timestamps_column]) / 1000.0 # Convert to seconds # Heuristic to test if timestamps are in Unix epoch length_in_seconds_of_a_10_year_experiment = 10 * 365 * 24 * 60 * 60 @@ -353,9 +369,9 @@ def extract_session_start_time( if configuration_file_path.is_file(): configuration_file = parse_fictrac_config(configuration_file_path) session_start_time = datetime.strptime(configuration_file.get("build_date", ""), "%b %d %Y") + # Set the time to midnight UTC from the extracted date return session_start_time.replace(hour=0, minute=0, second=0, microsecond=0, tzinfo=timezone.utc) - # If neither of the the methods above work, return None return None From 3f103e42527ab40055e185962b0ecb9e6091a115 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 31 Oct 2023 13:23:59 +0100 Subject: [PATCH 8/8] spelling --- .../datainterfaces/behavior/fictrac/fictracdatainterface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py index a43f35190..a7d163011 100644 --- a/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py +++ b/src/neuroconv/datainterfaces/behavior/fictrac/fictracdatainterface.py @@ -334,7 +334,7 @@ def extract_session_start_time( - For video file sources (.avi, .mp4, etc), the session start time corresponds to the time when the FicTrac analysis commenced. That is, the session start time reflects the analysis time rather than the actual start of the experiment. - - For camera sources (such as PGR or Basler), the sesion start time is either the time reported by the camera + - For camera sources (such as PGR or Basler), the session start time is either the time reported by the camera or the system time if the camera's SDK does not provide timestamps to Fictrac. In both cases, this time is the experiment start time, barring synchronization issues.