From abbe8f18519d798dc1841224134fc0866216ee55 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Wed, 13 Jul 2022 09:25:55 -0400 Subject: [PATCH 01/16] saving state --- nwbinspector/checks/tables.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nwbinspector/checks/tables.py b/nwbinspector/checks/tables.py index 62ba6fde4..205c309e1 100644 --- a/nwbinspector/checks/tables.py +++ b/nwbinspector/checks/tables.py @@ -148,9 +148,7 @@ def check_column_binary_capability(table: DynamicTable, nelems: int = 200): @register_check(importance=Importance.BEST_PRACTICE_SUGGESTION, neurodata_type=DynamicTable) def check_single_row( - table: DynamicTable, - exclude_types: Optional[list] = (Units,), - exclude_names: Optional[List[str]] = ("electrodes",), + table: DynamicTable, exclude_types: Optional[list] = (Units,), exclude_names: Optional[List[str]] = ("electrodes",), ): """ Check if DynamicTable has only a single row; may be better represented by another data type. @@ -183,3 +181,9 @@ def check_table_values_for_dict(table: DynamicTable, nelems: int = 200): if is_string_json_loadable(string=string): message += " This string is also JSON loadable, so call `json.loads(...)` on the string to unpack." yield InspectorMessage(message=message) + + +@register_check(importance=Importance.BEST_PRACTICE_SUGGESTION, neurodata_type=DynamicTable) +def check_col_not_nan(table: DynamicTable, nelems: int = 200): + """Check if all of the values in a single column of a table are NaN.""" + pass # TODO From 2a6516b54aa8d5d824aef22928e4cad43f26333f Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Wed, 13 Jul 2022 10:30:43 -0400 Subject: [PATCH 02/16] added test and debug --- nwbinspector/checks/tables.py | 11 ++++- tests/unit_tests/test_tables.py | 75 ++++++++++++++++++++++++--------- 2 files changed, 66 insertions(+), 20 deletions(-) diff --git a/nwbinspector/checks/tables.py b/nwbinspector/checks/tables.py index 205c309e1..3dd259bad 100644 --- a/nwbinspector/checks/tables.py +++ b/nwbinspector/checks/tables.py @@ -186,4 +186,13 @@ def check_table_values_for_dict(table: DynamicTable, nelems: int = 200): @register_check(importance=Importance.BEST_PRACTICE_SUGGESTION, neurodata_type=DynamicTable) def check_col_not_nan(table: DynamicTable, nelems: int = 200): """Check if all of the values in a single column of a table are NaN.""" - pass # TODO + for column in table.columns: + if not hasattr(column, "data") or isinstance(column, VectorIndex) or isinstance(column.data[0], str): + continue + subindex_selection = np.unique(np.round(np.linspace(start=0, stop=column.shape[0] - 1, num=nelems)).astype(int)) + if np.any(~np.isnan(column[subindex_selection])): + continue + else: + yield InspectorMessage( + message=f"Column {column.name} has all NaN values. Consider removing it from the table." + ) diff --git a/tests/unit_tests/test_tables.py b/tests/unit_tests/test_tables.py index e998fff90..50dd5e9ed 100644 --- a/tests/unit_tests/test_tables.py +++ b/tests/unit_tests/test_tables.py @@ -17,6 +17,7 @@ check_column_binary_capability, check_single_row, check_table_values_for_dict, + check_col_not_nan, ) from nwbinspector.utils import get_package_version @@ -228,17 +229,13 @@ def test_check_single_row_pass(): def test_check_single_row_ignore_units(): - table = Units( - name="Units", # default name when building through nwbfile - ) + table = Units(name="Units",) # default name when building through nwbfile table.add_unit(spike_times=[1, 2, 3]) assert check_single_row(table=table) is None def test_check_single_row_ignore_electrodes(): - table = ElectrodeTable( - name="electrodes", # default name when building through nwbfile - ) + table = ElectrodeTable(name="electrodes",) # default name when building through nwbfile if get_package_version(name="pynwb") >= version.Version("2.1.0"): table.add_row( location="unknown", @@ -291,7 +288,7 @@ def test_check_table_values_for_dict_pass(): assert check_table_values_for_dict(table=table) is None -def test_check_table_values_for_dict(): +def test_check_table_values_for_dict_fail(): table = DynamicTable(name="test_table", description="") table.add_column(name="test_column", description="") table.add_row(test_column=str(dict(a=1))) @@ -308,19 +305,59 @@ def test_check_table_values_for_dict(): ) -def test_check_table_values_for_dict_json_case(): +def test_check_table_values_for_dict_json_case_fail(): table = DynamicTable(name="test_table", description="") table.add_column(name="test_column", description="") table.add_row(test_column=json.dumps(dict(a=1))) - assert check_table_values_for_dict(table=table)[0] == InspectorMessage( - message=( - "The column 'test_column' contains a string value that contains a dictionary! Please unpack " - "dictionaries as additional rows or columns of the table. This string is also JSON loadable, so call " - "`json.loads(...)` on the string to unpack." + assert check_table_values_for_dict(table=table) == [ + InspectorMessage( + message=( + "The column 'test_column' contains a string value that contains a dictionary! Please unpack " + "dictionaries as additional rows or columns of the table. This string is also JSON loadable, so call " + "`json.loads(...)` on the string to unpack." + ), + importance=Importance.BEST_PRACTICE_VIOLATION, + check_function_name="check_table_values_for_dict", + object_type="DynamicTable", + object_name="test_table", + location="/", + ) + ] + + +def test_check_col_not_nan_pass(): + table = DynamicTable(name="test_table", description="") + for name in ["test_column_not_nan", "test_column_string"]: + table.add_column(name=name, description="") + table.add_row(test_column_not_nan=1.0, test_column_string="abc") + assert check_col_not_nan(table=table) is None + + +def test_check_col_not_nan_fail(): + table = DynamicTable(name="test_table", description="") + for name in ["test_column_not_nan_1", "test_column_nan_1", "test_column_not_nan_2", "test_column_nan_2"]: + table.add_column(name=name, description="") + for _ in range(400): + table.add_row( + test_column_not_nan_1=1.0, test_column_nan_1=np.nan, test_column_not_nan_2=1.0, test_column_nan_2=np.nan + ) + assert check_col_not_nan(table=table) == [ + InspectorMessage( + message="Column test_column_nan_1 has all NaN values. Consider removing it from the table.", + importance=Importance.BEST_PRACTICE_SUGGESTION, + check_function_name="check_col_not_nan", + object_type="DynamicTable", + object_name="test_table", + location="/", + file_path=None, ), - importance=Importance.BEST_PRACTICE_VIOLATION, - check_function_name="check_table_values_for_dict", - object_type="DynamicTable", - object_name="test_table", - location="/", - ) + InspectorMessage( + message="Column test_column_nan_2 has all NaN values. Consider removing it from the table.", + importance=Importance.BEST_PRACTICE_SUGGESTION, + check_function_name="check_col_not_nan", + object_type="DynamicTable", + object_name="test_table", + location="/", + file_path=None, + ), + ] From a322c047727099c77fab056e8d23e4c1207491ee Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Jul 2022 14:35:22 +0000 Subject: [PATCH 03/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nwbinspector/checks/tables.py | 4 +++- tests/unit_tests/test_tables.py | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/nwbinspector/checks/tables.py b/nwbinspector/checks/tables.py index 3dd259bad..bf02458b0 100644 --- a/nwbinspector/checks/tables.py +++ b/nwbinspector/checks/tables.py @@ -148,7 +148,9 @@ def check_column_binary_capability(table: DynamicTable, nelems: int = 200): @register_check(importance=Importance.BEST_PRACTICE_SUGGESTION, neurodata_type=DynamicTable) def check_single_row( - table: DynamicTable, exclude_types: Optional[list] = (Units,), exclude_names: Optional[List[str]] = ("electrodes",), + table: DynamicTable, + exclude_types: Optional[list] = (Units,), + exclude_names: Optional[List[str]] = ("electrodes",), ): """ Check if DynamicTable has only a single row; may be better represented by another data type. diff --git a/tests/unit_tests/test_tables.py b/tests/unit_tests/test_tables.py index 50dd5e9ed..58b0f48c6 100644 --- a/tests/unit_tests/test_tables.py +++ b/tests/unit_tests/test_tables.py @@ -229,13 +229,17 @@ def test_check_single_row_pass(): def test_check_single_row_ignore_units(): - table = Units(name="Units",) # default name when building through nwbfile + table = Units( + name="Units", + ) # default name when building through nwbfile table.add_unit(spike_times=[1, 2, 3]) assert check_single_row(table=table) is None def test_check_single_row_ignore_electrodes(): - table = ElectrodeTable(name="electrodes",) # default name when building through nwbfile + table = ElectrodeTable( + name="electrodes", + ) # default name when building through nwbfile if get_package_version(name="pynwb") >= version.Version("2.1.0"): table.add_row( location="unknown", From db1b426b9745f7c24ab6066a7176c499d78635a2 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Wed, 13 Jul 2022 12:13:47 -0400 Subject: [PATCH 04/16] refactor logic call --- nwbinspector/checks/tables.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nwbinspector/checks/tables.py b/nwbinspector/checks/tables.py index 3dd259bad..206aebe4c 100644 --- a/nwbinspector/checks/tables.py +++ b/nwbinspector/checks/tables.py @@ -190,9 +190,7 @@ def check_col_not_nan(table: DynamicTable, nelems: int = 200): if not hasattr(column, "data") or isinstance(column, VectorIndex) or isinstance(column.data[0], str): continue subindex_selection = np.unique(np.round(np.linspace(start=0, stop=column.shape[0] - 1, num=nelems)).astype(int)) - if np.any(~np.isnan(column[subindex_selection])): - continue - else: + if all(np.isnan(column[subindex_selection])): yield InspectorMessage( message=f"Column {column.name} has all NaN values. Consider removing it from the table." ) From 29a171d57bea6e67c63fb5e1275e5324b1d6d385 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Wed, 13 Jul 2022 13:46:05 -0400 Subject: [PATCH 05/16] add early data access skip --- nwbinspector/checks/tables.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nwbinspector/checks/tables.py b/nwbinspector/checks/tables.py index b66e8ce31..9ad10181a 100644 --- a/nwbinspector/checks/tables.py +++ b/nwbinspector/checks/tables.py @@ -148,9 +148,7 @@ def check_column_binary_capability(table: DynamicTable, nelems: int = 200): @register_check(importance=Importance.BEST_PRACTICE_SUGGESTION, neurodata_type=DynamicTable) def check_single_row( - table: DynamicTable, - exclude_types: Optional[list] = (Units,), - exclude_names: Optional[List[str]] = ("electrodes",), + table: DynamicTable, exclude_types: Optional[list] = (Units,), exclude_names: Optional[List[str]] = ("electrodes",), ): """ Check if DynamicTable has only a single row; may be better represented by another data type. @@ -191,6 +189,9 @@ def check_col_not_nan(table: DynamicTable, nelems: int = 200): for column in table.columns: if not hasattr(column, "data") or isinstance(column, VectorIndex) or isinstance(column.data[0], str): continue + if not all(np.isnan(column[:nelems])): + continue + subindex_selection = np.unique(np.round(np.linspace(start=0, stop=column.shape[0] - 1, num=nelems)).astype(int)) if all(np.isnan(column[subindex_selection])): yield InspectorMessage( From 2ac54f8c26f3748e825302b725c9bdd5058a7193 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Jul 2022 17:47:25 +0000 Subject: [PATCH 06/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nwbinspector/checks/tables.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nwbinspector/checks/tables.py b/nwbinspector/checks/tables.py index 9ad10181a..23b479360 100644 --- a/nwbinspector/checks/tables.py +++ b/nwbinspector/checks/tables.py @@ -148,7 +148,9 @@ def check_column_binary_capability(table: DynamicTable, nelems: int = 200): @register_check(importance=Importance.BEST_PRACTICE_SUGGESTION, neurodata_type=DynamicTable) def check_single_row( - table: DynamicTable, exclude_types: Optional[list] = (Units,), exclude_names: Optional[List[str]] = ("electrodes",), + table: DynamicTable, + exclude_types: Optional[list] = (Units,), + exclude_names: Optional[List[str]] = ("electrodes",), ): """ Check if DynamicTable has only a single row; may be better represented by another data type. From 73d4978dd6a3c1ef3be0eb268626ea382bb1fe4d Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Wed, 13 Jul 2022 13:52:28 -0400 Subject: [PATCH 07/16] add flatten for indexed cols --- nwbinspector/checks/tables.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nwbinspector/checks/tables.py b/nwbinspector/checks/tables.py index 9ad10181a..b171c611f 100644 --- a/nwbinspector/checks/tables.py +++ b/nwbinspector/checks/tables.py @@ -189,7 +189,7 @@ def check_col_not_nan(table: DynamicTable, nelems: int = 200): for column in table.columns: if not hasattr(column, "data") or isinstance(column, VectorIndex) or isinstance(column.data[0], str): continue - if not all(np.isnan(column[:nelems])): + if not all(np.isnan(column[:nelems]).flatten()): continue subindex_selection = np.unique(np.round(np.linspace(start=0, stop=column.shape[0] - 1, num=nelems)).astype(int)) From 48c5e602fa2e763939829c9c6afe48294183119c Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Wed, 13 Jul 2022 15:57:36 -0400 Subject: [PATCH 08/16] generalized to util function; added None slicing --- nwbinspector/checks/tables.py | 11 +++++------ nwbinspector/utils.py | 11 ++++++++++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/nwbinspector/checks/tables.py b/nwbinspector/checks/tables.py index c67dcfad9..2214808ff 100644 --- a/nwbinspector/checks/tables.py +++ b/nwbinspector/checks/tables.py @@ -13,6 +13,7 @@ is_ascending_series, is_dict_in_string, is_string_json_loadable, + get_uniform_indexes, ) @@ -148,9 +149,7 @@ def check_column_binary_capability(table: DynamicTable, nelems: int = 200): @register_check(importance=Importance.BEST_PRACTICE_SUGGESTION, neurodata_type=DynamicTable) def check_single_row( - table: DynamicTable, - exclude_types: Optional[list] = (Units,), - exclude_names: Optional[List[str]] = ("electrodes",), + table: DynamicTable, exclude_types: Optional[list] = (Units,), exclude_names: Optional[List[str]] = ("electrodes",), ): """ Check if DynamicTable has only a single row; may be better represented by another data type. @@ -186,15 +185,15 @@ def check_table_values_for_dict(table: DynamicTable, nelems: int = 200): @register_check(importance=Importance.BEST_PRACTICE_SUGGESTION, neurodata_type=DynamicTable) -def check_col_not_nan(table: DynamicTable, nelems: int = 200): +def check_col_not_nan(table: DynamicTable, nelems: Optional[int] = 200): """Check if all of the values in a single column of a table are NaN.""" for column in table.columns: if not hasattr(column, "data") or isinstance(column, VectorIndex) or isinstance(column.data[0], str): continue - if not all(np.isnan(column[:nelems]).flatten()): + if nelems is not None and not all(np.isnan(column[:nelems]).flatten()): continue - subindex_selection = np.unique(np.round(np.linspace(start=0, stop=column.shape[0] - 1, num=nelems)).astype(int)) + subindex_selection = get_uniform_indexes(length=column.shape[0], nelems=nelems) if all(np.isnan(column[subindex_selection])): yield InspectorMessage( message=f"Column {column.name} has all NaN values. Consider removing it from the table." diff --git a/nwbinspector/utils.py b/nwbinspector/utils.py index d37b1eea5..5c0f99b5f 100644 --- a/nwbinspector/utils.py +++ b/nwbinspector/utils.py @@ -57,6 +57,15 @@ def is_ascending_series(series: np.ndarray, nelems=None): return np.all(np.diff(series[:nelems]) > 0) +def get_uniform_indexes(length: int, nelems: Optional[int] = 200): + """General purpose function for generating nelems indices from 0 to length-1 with approximately equal gaps.""" + indexes = np.round(np.linspace(start=0, stop=length - 1, num=nelems)).astype(int) + if nelems is None or nelems >= length - 1: + return slice(0, None) + else: + return indexes + + def is_dict_in_string(string: str): """ Determine if the string value contains an encoded Python dictionary. @@ -127,7 +136,7 @@ def robust_s3_read( try: return command(*command_args, **command_kwargs) except OSError: # cannot curl request - sleep(0.1 * 2**retry) + sleep(0.1 * 2 ** retry) except Exception as exc: raise exc raise TimeoutError(f"Unable to complete the command ({command.__name__}) after {max_retries} attempts!") From b67f2c854f1b2a5cdda72269c1b4eca5a2e4ca1d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 13 Jul 2022 19:57:47 +0000 Subject: [PATCH 09/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nwbinspector/checks/tables.py | 4 +++- nwbinspector/utils.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nwbinspector/checks/tables.py b/nwbinspector/checks/tables.py index 2214808ff..31a8f2b0c 100644 --- a/nwbinspector/checks/tables.py +++ b/nwbinspector/checks/tables.py @@ -149,7 +149,9 @@ def check_column_binary_capability(table: DynamicTable, nelems: int = 200): @register_check(importance=Importance.BEST_PRACTICE_SUGGESTION, neurodata_type=DynamicTable) def check_single_row( - table: DynamicTable, exclude_types: Optional[list] = (Units,), exclude_names: Optional[List[str]] = ("electrodes",), + table: DynamicTable, + exclude_types: Optional[list] = (Units,), + exclude_names: Optional[List[str]] = ("electrodes",), ): """ Check if DynamicTable has only a single row; may be better represented by another data type. diff --git a/nwbinspector/utils.py b/nwbinspector/utils.py index 5c0f99b5f..4b4604a9a 100644 --- a/nwbinspector/utils.py +++ b/nwbinspector/utils.py @@ -136,7 +136,7 @@ def robust_s3_read( try: return command(*command_args, **command_kwargs) except OSError: # cannot curl request - sleep(0.1 * 2 ** retry) + sleep(0.1 * 2**retry) except Exception as exc: raise exc raise TimeoutError(f"Unable to complete the command ({command.__name__}) after {max_retries} attempts!") From 75a687e2736d41fca17b6325045e15f3cf4f816c Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Mon, 18 Jul 2022 15:09:58 -0400 Subject: [PATCH 10/16] swapped util to return only slice --- nwbinspector/checks/tables.py | 4 ++-- nwbinspector/utils.py | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/nwbinspector/checks/tables.py b/nwbinspector/checks/tables.py index 31a8f2b0c..926356e98 100644 --- a/nwbinspector/checks/tables.py +++ b/nwbinspector/checks/tables.py @@ -13,7 +13,7 @@ is_ascending_series, is_dict_in_string, is_string_json_loadable, - get_uniform_indexes, + safe_uniform_selection, ) @@ -195,7 +195,7 @@ def check_col_not_nan(table: DynamicTable, nelems: Optional[int] = 200): if nelems is not None and not all(np.isnan(column[:nelems]).flatten()): continue - subindex_selection = get_uniform_indexes(length=column.shape[0], nelems=nelems) + subindex_selection = safe_uniform_selection(length=column.shape[0], nelems=nelems) if all(np.isnan(column[subindex_selection])): yield InspectorMessage( message=f"Column {column.name} has all NaN values. Consider removing it from the table." diff --git a/nwbinspector/utils.py b/nwbinspector/utils.py index 4b4604a9a..3784f6b51 100644 --- a/nwbinspector/utils.py +++ b/nwbinspector/utils.py @@ -57,13 +57,11 @@ def is_ascending_series(series: np.ndarray, nelems=None): return np.all(np.diff(series[:nelems]) > 0) -def get_uniform_indexes(length: int, nelems: Optional[int] = 200): - """General purpose function for generating nelems indices from 0 to length-1 with approximately equal gaps.""" - indexes = np.round(np.linspace(start=0, stop=length - 1, num=nelems)).astype(int) - if nelems is None or nelems >= length - 1: +def safe_uniform_selection(length: int, nelems: Optional[int] = 200) -> slice: + """General purpose function for safely generating an evenly spaced slice .""" + if nelems is None: return slice(0, None) - else: - return indexes + return slice(0, length, np.ceil(length/nelems).astype(int)) def is_dict_in_string(string: str): From 62350dbf6811f1a8b39d950ad4028ff7bb91c383 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 19:10:11 +0000 Subject: [PATCH 11/16] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- nwbinspector/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nwbinspector/utils.py b/nwbinspector/utils.py index 3784f6b51..558d879b2 100644 --- a/nwbinspector/utils.py +++ b/nwbinspector/utils.py @@ -61,7 +61,7 @@ def safe_uniform_selection(length: int, nelems: Optional[int] = 200) -> slice: """General purpose function for safely generating an evenly spaced slice .""" if nelems is None: return slice(0, None) - return slice(0, length, np.ceil(length/nelems).astype(int)) + return slice(0, length, np.ceil(length / nelems).astype(int)) def is_dict_in_string(string: str): From 1e83c6b8ec2d936173b90b762e9de9861bae1df9 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 18 Jul 2022 16:44:11 -0400 Subject: [PATCH 12/16] Update nwbinspector/utils.py Co-authored-by: Ben Dichter --- nwbinspector/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nwbinspector/utils.py b/nwbinspector/utils.py index bdf82688e..a97b9f75f 100644 --- a/nwbinspector/utils.py +++ b/nwbinspector/utils.py @@ -59,9 +59,7 @@ def is_ascending_series(series: np.ndarray, nelems=None): def safe_uniform_selection(length: int, nelems: Optional[int] = 200) -> slice: """General purpose function for safely generating an evenly spaced slice .""" - if nelems is None: - return slice(0, None) - return slice(0, length, np.ceil(length / nelems).astype(int)) + return slice(0, None, np.ceil(length / nelems).astype(int) if nelems else None) def is_dict_in_string(string: str): From 09b0474267d7209e7be8a927ce7b9a2fba650531 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 19 Jul 2022 09:56:00 -0400 Subject: [PATCH 13/16] Update nwbinspector/checks/tables.py Co-authored-by: Ben Dichter --- nwbinspector/checks/tables.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nwbinspector/checks/tables.py b/nwbinspector/checks/tables.py index e3d3de88c..eebd4d4d8 100644 --- a/nwbinspector/checks/tables.py +++ b/nwbinspector/checks/tables.py @@ -200,8 +200,7 @@ def check_col_not_nan(table: DynamicTable, nelems: Optional[int] = 200): if nelems is not None and not all(np.isnan(column[:nelems]).flatten()): continue - subindex_selection = safe_uniform_selection(length=column.shape[0], nelems=nelems) - if all(np.isnan(column[subindex_selection])): + if all(np.isnan(column[slice(0, None, np.ceil(length / nelems).astype(int) if nelems else None)])): yield InspectorMessage( message=f"Column {column.name} has all NaN values. Consider removing it from the table." ) From 10c2073cbfaf1002b1f80e7c76049d6a51659809 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 19 Jul 2022 09:56:06 -0400 Subject: [PATCH 14/16] Update nwbinspector/utils.py Co-authored-by: Ben Dichter --- nwbinspector/utils.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/nwbinspector/utils.py b/nwbinspector/utils.py index 4e886186b..48aa6a84c 100644 --- a/nwbinspector/utils.py +++ b/nwbinspector/utils.py @@ -90,11 +90,6 @@ def is_ascending_series(series: Union[h5py.Dataset, ArrayLike], nelems=None): return np.all(np.diff(series[:nelems]) > 0) # already in memory, no need to cache -def safe_uniform_selection(length: int, nelems: Optional[int] = 200) -> slice: - """General purpose function for safely generating an evenly spaced slice .""" - return slice(0, None, np.ceil(length / nelems).astype(int) if nelems else None) - - def is_dict_in_string(string: str): """ Determine if the string value contains an encoded Python dictionary. From 41056ea96048f2f740e984a056a7aac26d0af4ec Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Tue, 19 Jul 2022 09:56:11 -0400 Subject: [PATCH 15/16] Update nwbinspector/checks/tables.py Co-authored-by: Ben Dichter --- nwbinspector/checks/tables.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nwbinspector/checks/tables.py b/nwbinspector/checks/tables.py index eebd4d4d8..adcad1c2a 100644 --- a/nwbinspector/checks/tables.py +++ b/nwbinspector/checks/tables.py @@ -14,7 +14,6 @@ is_ascending_series, is_dict_in_string, is_string_json_loadable, - safe_uniform_selection, ) From 8def458cd592bccf08a8eb1a24e5dc61e51d6996 Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Tue, 19 Jul 2022 10:28:16 -0400 Subject: [PATCH 16/16] debug --- nwbinspector/checks/tables.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nwbinspector/checks/tables.py b/nwbinspector/checks/tables.py index adcad1c2a..df83a3bb5 100644 --- a/nwbinspector/checks/tables.py +++ b/nwbinspector/checks/tables.py @@ -199,7 +199,11 @@ def check_col_not_nan(table: DynamicTable, nelems: Optional[int] = 200): if nelems is not None and not all(np.isnan(column[:nelems]).flatten()): continue - if all(np.isnan(column[slice(0, None, np.ceil(length / nelems).astype(int) if nelems else None)])): + if all( + np.isnan( + column[slice(0, None, np.ceil(len(column.data) / nelems).astype(int) if nelems else None)] + ).flatten() + ): yield InspectorMessage( message=f"Column {column.name} has all NaN values. Consider removing it from the table." )