diff --git a/docs/challenges/Spatial Parameter/EgaitAddidas2014/index.rst b/docs/challenges/Spatial Parameter/EgaitAddidas2014/index.rst index a401041..fff8f83 100644 --- a/docs/challenges/Spatial Parameter/EgaitAddidas2014/index.rst +++ b/docs/challenges/Spatial Parameter/EgaitAddidas2014/index.rst @@ -1,5 +1,5 @@ EgaitAdidas2014 - Spatial Parameters -============================================= +==================================== .. toctree:: :glob: diff --git a/docs/challenges/Stride Segmentation/EgaitSegmentationValidation2014/index.rst b/docs/challenges/Stride Segmentation/EgaitSegmentationValidation2014/index.rst new file mode 100644 index 0000000..2e6cd48 --- /dev/null +++ b/docs/challenges/Stride Segmentation/EgaitSegmentationValidation2014/index.rst @@ -0,0 +1,30 @@ +EgaitSegmentationValidation2014 +=============================== + +.. toctree:: + :glob: + :maxdepth: 1 + + Link to Result Page + +.. automodule:: gaitmap_challenges.stride_segmentation.egait_segmentation_validation_2014 + :no-members: + :no-inherited-members: + +Code Objects +------------ + +.. currentmodule:: gaitmap_challenges.stride_segmentation.egait_segmentation_validation_2014 + +.. autosummary:: + :toctree: generated/ + :template: class.rst + + Challenge + ChallengeDataset + +.. autosummary:: + :toctree: generated/ + :template: function.rst + + final_scorer diff --git a/docs/challenges/Stride Segmentation/EgaitSegmentationValidation2014/index.ipynb b/docs/challenges/Stride Segmentation/EgaitSegmentationValidation2014/results.ipynb similarity index 100% rename from docs/challenges/Stride Segmentation/EgaitSegmentationValidation2014/index.ipynb rename to docs/challenges/Stride Segmentation/EgaitSegmentationValidation2014/results.ipynb diff --git a/docs/challenges/Stride Segmentation/EgaitSegmentationValidation2014OriginalLabel/index.rst b/docs/challenges/Stride Segmentation/EgaitSegmentationValidation2014OriginalLabel/index.rst new file mode 100644 index 0000000..3c90b0f --- /dev/null +++ b/docs/challenges/Stride Segmentation/EgaitSegmentationValidation2014OriginalLabel/index.rst @@ -0,0 +1,30 @@ +EgaitSegmentationValidation2014 - Original Labels +================================================= + +.. toctree:: + :glob: + :maxdepth: 1 + + Link to Result Page + +.. automodule:: gaitmap_challenges.stride_segmentation.egait_segmentation_validation_2014_original_label + :no-members: + :no-inherited-members: + +Code Objects +------------ + +.. currentmodule:: gaitmap_challenges.stride_segmentation.egait_segmentation_validation_2014_original_label + +.. autosummary:: + :toctree: generated/ + :template: class.rst + + Challenge + ChallengeDataset + +.. autosummary:: + :toctree: generated/ + :template: function.rst + + final_scorer diff --git a/docs/challenges/Stride Segmentation/EgaitSegmentationValidation2014OriginalLabel/index.ipynb b/docs/challenges/Stride Segmentation/EgaitSegmentationValidation2014OriginalLabel/results.ipynb similarity index 100% rename from docs/challenges/Stride Segmentation/EgaitSegmentationValidation2014OriginalLabel/index.ipynb rename to docs/challenges/Stride Segmentation/EgaitSegmentationValidation2014OriginalLabel/results.ipynb diff --git a/docs/challenges/index.rst b/docs/challenges/index.rst index 7d6a7f5..e81bf31 100644 --- a/docs/challenges/index.rst +++ b/docs/challenges/index.rst @@ -1,6 +1,10 @@ Challenges ========== +Below you can find links to all challenges for which we currently have results generated. +Click on any of the challenges below, you should get a detailed description of the challenge and a link to +the result page. + .. toctree:: :glob: :maxdepth: 2 diff --git a/gaitmap-challenges/gaitmap_challenges/stride_segmentation/egait_segmentation_validation_2014.py b/gaitmap-challenges/gaitmap_challenges/stride_segmentation/egait_segmentation_validation_2014.py index 372a6b9..e27e946 100644 --- a/gaitmap-challenges/gaitmap_challenges/stride_segmentation/egait_segmentation_validation_2014.py +++ b/gaitmap-challenges/gaitmap_challenges/stride_segmentation/egait_segmentation_validation_2014.py @@ -1,10 +1,68 @@ +"""A challenge for stride segmentation from continuous IMU data @Lab and in simulated @home environments. + +General Information +------------------- + +Dataset + The `Egait Segmentation Validation 2014 `_ dataset [1]_ is used for this challenge + (`usage example `_, `download `_). + It contains 4x10m gait tests from 30 participants (10 controls, 10 Parkinson's disease, 10 geriatric patients) and + 15 simulated @home tests from 5 participants per cohort (5 controls, 5 Parkinson's disease, 5 geriatric patients ). +Sensor System + All participants wore Shimmer2R (102.4 Hz) IMUs laterally on both shoes. +Reference System + All trials where recorded with a video camera to make the original annotations (not used in this challenge). + In addition, the data was expert labeled based on the raw gyro data. + The original annotations explicitly excluded turns and stairs. + The new annotations included all gait like movements (turns, stairs, etc.). + This challenge uses the new annotations. + For a challenge using the original annotations, see + :mod:`~gaitmap_challenges.stride_segmentation.egait_segmentation_validation_2014_original_label`. + All annotations where performed following the stride definition by [1]_. + +Implementation Recommendations +------------------------------ + +This challenge version (using the new annotations) expects algorithms to be able to segment all types of strides +including turns and stairs. +This means algorithms likely need be more sensitive. + +The data does only contain gait and no other movements. +This should make this a relatively easy dataset for gait segmentation. + +References +---------- +.. [1] Barth, Jens, Cäcilia Oberndorfer, Cristian Pasluosta, Samuel Schülein, Heiko Gassner, Samuel Reinfelder, + Patrick Kugler, et al. “Stride Segmentation during Free Walk Movements Using Multi-Dimensional Subsequence + Dynamic Time Warping on Inertial Sensor Data.” Sensors (Switzerland) 15, no. 3 (March 17, 2015): 6419-40. + https://doi.org/10.3390/s150306419. + +.. _dataset_info: https://www.mad.tf.fau.de/research/datasets/#collapse_18 +.. _datasets_example: + https://mad-lab-fau.github.io/gaitmap-datasets/auto_examples/egait_segmentation_validation_2014.html +.. _dataset_download: https://www.mad.tf.fau.de/research/datasets/#collapse_18 + +""" from dataclasses import dataclass, field from functools import partial from pathlib import Path -from typing import Any, ClassVar, Dict, Iterator, List, Literal, Optional, TypedDict, Union +from typing import ( + Any, + ClassVar, + Dict, + Iterator, + List, + Literal, + Optional, + TypedDict, + Union, +) import pandas as pd -from gaitmap.evaluation_utils import evaluate_segmented_stride_list, precision_recall_f1_score +from gaitmap.evaluation_utils import ( + evaluate_segmented_stride_list, + precision_recall_f1_score, +) from gaitmap_datasets import EgaitSegmentationValidation2014 from sklearn.model_selection import BaseCrossValidator, StratifiedKFold from tpcp import Pipeline @@ -28,11 +86,47 @@ SensorNames = Literal["left_sensor", "right_sensor"] -def _final_scorer(pipeline: Pipeline, datapoint: EgaitSegmentationValidation2014, tolerance_s: float = 0.03): +def final_scorer( + pipeline: Pipeline, + datapoint: EgaitSegmentationValidation2014, + tolerance_s: float = 0.03, + use_original_labels: bool = False, +): + """Score a pipeline build for segmentation challenges of the Egait Segmentation Validation 2014 dataset. + + It compares the reference stride list (either the original or the new one) with the stride list returned by the + pipeline via the `stride_list_` attribute. + + It calculates the precision, recall and f1 score for the stride segmentation. + + Parameters + ---------- + pipeline + The pipeline to score. + The pipeline needs to have a `stride_list_` attribute that contains the segmented stride list after running. + datapoint + The datapoint of the EgaitSegmentationValidation2014 dataset. + tolerance_s + The tolerance in seconds that is allowed between the calculated stride borders and the reference stride + borders. + Both, start and end labels need to be within the tolerance to be considered a match. + use_original_labels + If True, the original stride labels are used for the reference stride list. + The original labels only contain straight strides. + If False, the new labels are used for the reference stride list. + The new labels contain all strides including turns and stairs. + + See Also + -------- + gaitmap.evaluation_utils.evaluate_segmented_stride_list + + """ results = pipeline.safe_run(datapoint) + reference = datapoint.segmented_stride_list_original_ if use_original_labels else datapoint.segmented_stride_list_ + matched_stride_list = evaluate_segmented_stride_list( - ground_truth=datapoint.segmented_stride_list_, + ground_truth=reference, segmented_stride_list=results.stride_list_, tolerance=int(tolerance_s * datapoint.sampling_rate_hz), ) @@ -55,6 +149,45 @@ class ResultType(TypedDict): @dataclass(repr=False) class Challenge(BaseChallenge): + """The EgaitSegmentation Validation Challenge. + + This challenge uses the new labels by default. + + Parameters + ---------- + dataset + A instance of :class:`~gaitmap_datasets.EgaitSegmentationValidation2014` or a path to a directory containing + the dataset. + cv_iterator + A cross-validation iterator or the number of folds to use. + cv_params + Additional parameters to pass to the tpcp cross-validation function. + + Other Parameters + ---------------- + match_tolerance_s + (Class Constant) The tolerance in seconds that is allowed between the calculated stride borders and the + reference stride borders. + use_original_labels + (Class Constant) If True, the original stride labels are used for the reference stride list. + The original labels only contain straight strides. + If False, the new labels are used for the reference stride list. + The new labels contain all strides including turns and stairs. + + Attributes + ---------- + cv_results_ + The results of the cross-validation. + This can be passed directly to the pandas DataFrame constructor to get a dataframe with the results. + + See Also + -------- + gaitmap_challenges.challenge_base.BaseChallenge : For common parameters and attributes of all challenges. + gaitmap_challenges.stride_segmentation.egait_segmentation_validation_2014_original_label.Challenge : The same + challenge, but with the original labels + + """ + dataset: Optional[Union[str, Path, ChallengeDataset]] cv_iterator: Optional[Union[int, BaseCrossValidator, Iterator]] = StratifiedKFold(n_splits=5) cv_params: Optional[Dict] = None @@ -93,7 +226,11 @@ def _resolve_dataset(self): @classmethod def get_scorer(cls): - return partial(_final_scorer, tolerance_s=cls.match_tolerance_s) + return partial( + final_scorer, + tolerance_s=cls.match_tolerance_s, + use_original_labels=cls.use_original_labels, + ) @classmethod def get_imu_data( @@ -134,4 +271,4 @@ def load_core_results(cls, folder_path) -> ResultType: } -__all__ = ["Challenge", "ChallengeDataset", "ResultType", "SensorNames"] +__all__ = ["Challenge", "ChallengeDataset", "ResultType", "SensorNames", "final_scorer"] diff --git a/gaitmap-challenges/gaitmap_challenges/stride_segmentation/egait_segmentation_validation_2014_original_label.py b/gaitmap-challenges/gaitmap_challenges/stride_segmentation/egait_segmentation_validation_2014_original_label.py index b32da6e..4698482 100644 --- a/gaitmap-challenges/gaitmap_challenges/stride_segmentation/egait_segmentation_validation_2014_original_label.py +++ b/gaitmap-challenges/gaitmap_challenges/stride_segmentation/egait_segmentation_validation_2014_original_label.py @@ -1,3 +1,47 @@ +"""A challenge for stride segmentation from continuous IMU data focused on straight strides. + +General Information +------------------- + +Dataset + The `Egait Segmentation Validation 2014 `_ dataset [1]_ is used for this challenge + (`usage example `_, `download `_). + It contains 4x10m gait tests from 30 participants (10 controls, 10 Parkinson's disease, 10 geriatric patients) and + 15 simulated @home tests from 5 participants per cohort (5 controls, 5 Parkinson's disease, 5 geriatric patients ). +Sensor System + All participants wore Shimmer2R (102.4 Hz) IMUs laterally on both shoes. +Reference System + All trials where recorded with a video camera to support labeling of the strides in the raw data. + The camera data was used to remove turns and stair strides from the annotations. + Another set of annotations exists, that contains all strides including turns and stairs. + All annotations where performed following the stride definition by [1]_. + However, this challenge used the original annotations that **excluded** turns and stairs. + For an equivalent challenge using the new annotations, see + :mod:`~gaitmap_challenges.stride_segmentation.egait_segmentation_validation_2014`. + +Implementation Recommendations +------------------------------ + +This challenge version uses the original annotations that **excluded** turns and stairs. +In result, the validation favors algorithms that are very specific to straight strides. +Algorithms that naturally have a higher sensitivity and also detect turns and stairs will be penalized. + +Overall the data only contains gait and no other movements. +This should make this a relatively easy dataset for gait segmentation. + +References +---------- +.. [1] Barth, Jens, Cäcilia Oberndorfer, Cristian Pasluosta, Samuel Schülein, Heiko Gassner, Samuel Reinfelder, + Patrick Kugler, et al. “Stride Segmentation during Free Walk Movements Using Multi-Dimensional Subsequence + Dynamic Time Warping on Inertial Sensor Data.” Sensors (Switzerland) 15, no. 3 (March 17, 2015): 6419-40. + https://doi.org/10.3390/s150306419. + +.. _dataset_info: https://www.mad.tf.fau.de/research/datasets/#collapse_18 +.. _datasets_example: + https://mad-lab-fau.github.io/gaitmap-datasets/auto_examples/egait_segmentation_validation_2014.html +.. _dataset_download: https://www.mad.tf.fau.de/research/datasets/#collapse_18 + +""" from gaitmap_challenges.stride_segmentation.egait_segmentation_validation_2014 import ( Challenge as BaseChallenge, ) @@ -5,11 +49,54 @@ ChallengeDataset, ResultType, SensorNames, + final_scorer, ) class Challenge(BaseChallenge): + """The EgaitSegmentation Validation Challenge using the original labels. + + This challenge is identical to + :class:`~gaitmap_challenges.stride_segmentation.egait_segmentation_validation_2014.Challenge`, + with the exception that `use_original_labels` is set to True by default. + This means that the original labels are used for the reference stride list. + + Parameters + ---------- + dataset + A instance of :class:`~gaitmap_datasets.EgaitSegmentationValidation2014` or a path to a directory containing + the dataset. + cv_iterator + A cross-validation iterator or the number of folds to use. + cv_params + Additional parameters to pass to the tpcp cross-validation function. + + Other Parameters + ---------------- + match_tolerance_s + (Class Constant) The tolerance in seconds that is allowed between the calculated stride borders and the + reference stride borders. + use_original_labels + (Class Constant) If True, the original stride labels are used for the reference stride list. + The original labels only contain straight strides. + If False, the new labels are used for the reference stride list. + The new labels contain all strides including turns and stairs. + + Attributes + ---------- + cv_results_ + The results of the cross-validation. + This can be passed directly to the pandas DataFrame constructor to get a dataframe with the results. + + See Also + -------- + gaitmap_challenges.challenge_base.BaseChallenge : For common parameters and attributes of all challenges. + gaitmap_challenges.stride_segmentation.egait_segmentation_validation_2014.Challenge : The same challenge, but with + the new labels + + """ + use_original_labels = True -__all__ = ["Challenge", "ChallengeDataset", "ResultType", "SensorNames"] +__all__ = ["Challenge", "ChallengeDataset", "ResultType", "SensorNames", "final_scorer"]