From e92587141de8eb2ef1582428ce3a3b29dcafa512 Mon Sep 17 00:00:00 2001 From: MikeG Date: Mon, 27 May 2024 11:28:25 +0200 Subject: [PATCH] Quell warnings with new versions of pandas (#82) * clean up warnings in tests * ignore warnings from `nptyping` and `openpyxl` --- atlas_densities/app/mtype_densities.py | 2 +- .../combination/markers_combinator.py | 15 ++++++--- atlas_densities/densities/excel_reader.py | 4 +-- .../densities/measurement_to_density.py | 1 - atlas_densities/densities/utils.py | 2 +- pyproject.toml | 9 +++++ tests/app/test_cell_densities.py | 5 ++- tests/app/test_mtype_densities.py | 33 ++++++++++--------- tests/densities/test_excel_reader.py | 31 ++++++++++------- ..._inhibitory_neuron_density_optimization.py | 18 +++++----- .../densities/test_measurement_to_density.py | 25 ++++++++------ .../test_mtype_densities_from_composition.py | 9 ++--- .../test_mtype_densities_from_profiles.py | 26 +++++++++------ .../test_optimization_initialization.py | 17 +++++----- 14 files changed, 119 insertions(+), 78 deletions(-) create mode 100644 pyproject.toml diff --git a/atlas_densities/app/mtype_densities.py b/atlas_densities/app/mtype_densities.py index 5aa5dd2..f221e1d 100644 --- a/atlas_densities/app/mtype_densities.py +++ b/atlas_densities/app/mtype_densities.py @@ -408,7 +408,7 @@ def create_from_composition( def _load_neuronal_mtype_taxonomy(filename: str) -> pd.DataFrame: """Loads the taxonomy tsv file into a dataframe""" - return pd.read_csv(filename, header=0, delim_whitespace=True) + return pd.read_csv(filename, header=0, sep=r"\s+") def _validate_mtype_taxonomy(taxonomy: pd.DataFrame) -> None: diff --git a/atlas_densities/combination/markers_combinator.py b/atlas_densities/combination/markers_combinator.py index ea61764..882ebef 100644 --- a/atlas_densities/combination/markers_combinator.py +++ b/atlas_densities/combination/markers_combinator.py @@ -127,14 +127,21 @@ def compute_voxel_count( # 'A Cell Atlas for the Mouse Brain' in that we use the weights 1.0 / E_marker normalized by # their sums instead of the coefficients C_marker = 1.0 / (E_marker * N_marker). combination_data.sort_index(inplace=True) - intensities = combination_data.groupby("cellType").apply( - lambda x: np.average( + + def func(x): + return np.average( np.array(list(x.volume)), - weights=1.0 / np.array(list(x["averageExpressionIntensity"])), + weights=1.0 / x["averageExpressionIntensity"].to_numpy(), axis=0, ) + + kwargs = {} + if tuple(pd.__version__.split(".")) > ("2", "2", "0"): + kwargs["include_groups"] = False + + intensities = ( + combination_data.groupby("cellType").apply(func, **kwargs).rename("intensity").to_frame() ) - intensities = pd.DataFrame(intensities, columns=["intensity"]) intensities["proportion"] = proportions # Average overall glia intensity diff --git a/atlas_densities/densities/excel_reader.py b/atlas_densities/densities/excel_reader.py index 9a336b1..7f1d85e 100644 --- a/atlas_densities/densities/excel_reader.py +++ b/atlas_densities/densities/excel_reader.py @@ -164,7 +164,7 @@ def compute_kim_et_al_neuron_densities( engine="openpyxl", ).set_index("ROI") for acronym in kim_et_al_mmc3.index: - full_name = kim_et_al_mmc3.loc[acronym][0] + full_name = kim_et_al_mmc3.loc[acronym, "full_name"] if not isinstance(full_name, str) or not full_name: warn( f"Region with acronym {acronym} has no valid full name. " @@ -524,7 +524,7 @@ def read_homogenous_neuron_type_regions(measurements_path: str | "Path") -> pd.D ) excitatory_mask = homogenous_regions_worksheet["comment"] == "Purely excitatory region" homogenous_regions_worksheet["cell_type"] = "inhibitory" - homogenous_regions_worksheet["cell_type"][excitatory_mask] = "excitatory" + homogenous_regions_worksheet.loc[excitatory_mask, "cell_type"] = "excitatory" homogenous_regions_worksheet.drop(columns=["comment"], inplace=True) return homogenous_regions_worksheet diff --git a/atlas_densities/densities/measurement_to_density.py b/atlas_densities/densities/measurement_to_density.py index d300304..257c032 100644 --- a/atlas_densities/densities/measurement_to_density.py +++ b/atlas_densities/densities/measurement_to_density.py @@ -277,7 +277,6 @@ def remove_unknown_regions( hierarchy_info: data frame returned by :func:`atlas_densities.densities.utils.get_hierarchy_info`. """ - pd.set_option("display.max_colwidth", None) indices_ids = measurements.index[ ~measurements["brain_region"].isin(hierarchy_info["brain_region"]) ] diff --git a/atlas_densities/densities/utils.py b/atlas_densities/densities/utils.py index 2251e04..2e76655 100644 --- a/atlas_densities/densities/utils.py +++ b/atlas_densities/densities/utils.py @@ -530,7 +530,7 @@ def compute_region_volumes( ) ids, counts = np.unique(annotation, return_counts=True) - result["id_volume"].update(pd.Series(counts * voxel_volume, index=ids)) + result.update({"id_volume": pd.Series(counts * voxel_volume, index=ids)}) volumes = [] for id_ in hierarchy_info.index: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6ed64e2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[tool.pytest.ini_options] +addopts = [ + "--import-mode=importlib", +] +filterwarnings = [ + "ignore::DeprecationWarning:nptyping", + "ignore::UserWarning:openpyxl", +] + diff --git a/tests/app/test_cell_densities.py b/tests/app/test_cell_densities.py index 4c8463a..8a05a1b 100644 --- a/tests/app/test_cell_densities.py +++ b/tests/app/test_cell_densities.py @@ -1,6 +1,7 @@ """test cell_densities""" import json +import warnings from pathlib import Path import numpy as np @@ -253,7 +254,9 @@ def _get_measurements_to_average_densities_result(runner, hierarchy_path, measur # fmt: on ] - return runner.invoke(tested.app, args) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return runner.invoke(tested.app, args) def test_measurements_to_average_densities(): diff --git a/tests/app/test_mtype_densities.py b/tests/app/test_mtype_densities.py index 1ca562f..576d5c8 100644 --- a/tests/app/test_mtype_densities.py +++ b/tests/app/test_mtype_densities.py @@ -1,6 +1,7 @@ """test app/mtype_densities""" import json import os +import warnings from copy import deepcopy from pathlib import Path @@ -27,21 +28,23 @@ def get_result_from_profiles(runner, td): - return runner.invoke( - tested.app, - [ - # fmt: off - "--log-output-path", td, - "create-from-profile", - "--annotation-path", "annotation.nrrd", - "--hierarchy-path", "hierarchy.json", - "--metadata-path", "metadata.json", - "--direction-vectors-path", "direction_vectors.nrrd", - "--mtypes-config-path", "config.yaml", - "--output-dir", "output_dir", - # fmt: on - ], - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return runner.invoke( + tested.app, + [ + # fmt: off + "--log-output-path", td, + "create-from-profile", + "--annotation-path", "annotation.nrrd", + "--hierarchy-path", "hierarchy.json", + "--metadata-path", "metadata.json", + "--direction-vectors-path", "direction_vectors.nrrd", + "--mtypes-config-path", "config.yaml", + "--output-dir", "output_dir", + # fmt: on + ], + ) def test_mtype_densities_from_profiles(tmp_path): diff --git a/tests/densities/test_excel_reader.py b/tests/densities/test_excel_reader.py index 1cf05d7..e6613eb 100644 --- a/tests/densities/test_excel_reader.py +++ b/tests/densities/test_excel_reader.py @@ -178,10 +178,11 @@ def check_non_negative_values(dataframe): def test_read_kim_et_al_neuron_densities(): - warnings.simplefilter("ignore") - dataframe = tested.read_kim_et_al_neuron_densities( - REGION_MAP, Path(MEASUREMENTS_PATH, "mmc3.xlsx") - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + dataframe = tested.read_kim_et_al_neuron_densities( + REGION_MAP, Path(MEASUREMENTS_PATH, "mmc3.xlsx") + ) check_column_names(dataframe) check_column_types(dataframe) @@ -331,13 +332,14 @@ def test_read_non_density_measurements(): def test_read_measurements(): - warnings.simplefilter("ignore") - dataframe = tested.read_measurements( - REGION_MAP, - Path(MEASUREMENTS_PATH, "mmc3.xlsx"), - Path(MEASUREMENTS_PATH, "gaba_papers.xlsx"), - Path(MEASUREMENTS_PATH, "non_density_measurements.csv"), - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + dataframe = tested.read_measurements( + REGION_MAP, + Path(MEASUREMENTS_PATH, "mmc3.xlsx"), + Path(MEASUREMENTS_PATH, "gaba_papers.xlsx"), + Path(MEASUREMENTS_PATH, "non_density_measurements.csv"), + ) check_column_names(dataframe) check_non_negative_values(dataframe) check_columns_na(dataframe) @@ -357,3 +359,10 @@ def test_read_measurements(): # TODO: one duplicate title because of a trailing endline character assert len(set(dataframe["source_title"])) == 48 assert len(set(dataframe["comment"])) >= 48 + + +def test_read_homogenous_neuron_type_regions(): + res = tested.read_homogenous_neuron_type_regions(MEASUREMENTS_PATH / "gaba_papers.xlsx") + assert len(res) == 60 + assert np.count_nonzero(res.cell_type == "excitatory") == 3 + assert np.count_nonzero(res.cell_type == "inhibitory") == 57 diff --git a/tests/densities/test_inhibitory_neuron_density_optimization.py b/tests/densities/test_inhibitory_neuron_density_optimization.py index 17d50dd..7ab0530 100644 --- a/tests/densities/test_inhibitory_neuron_density_optimization.py +++ b/tests/densities/test_inhibitory_neuron_density_optimization.py @@ -10,7 +10,7 @@ import pytest import atlas_densities.densities.inhibitory_neuron_densities_optimization as tested -from atlas_densities.exceptions import AtlasDensitiesError +from atlas_densities.exceptions import AtlasDensitiesError, AtlasDensitiesWarning def test_resize_average_densities(): @@ -322,13 +322,15 @@ def get_initialization_data_4(): def test_create_inhibitory_neuron_densities_4(): data = get_initialization_data_4() - densities = tested.create_inhibitory_neuron_densities( - data["hierarchy"], - data["annotation"], - data["voxel_volume"], - data["neuron_density"], - data["average_densities"], - ) + + with pytest.warns(AtlasDensitiesWarning): + densities = tested.create_inhibitory_neuron_densities( + data["hierarchy"], + data["annotation"], + data["voxel_volume"], + data["neuron_density"], + data["average_densities"], + ) expected = { "gad67+": np.array([[[0.5, 0.5, 1.0, 1.0, 1.5, 1.5]]]), diff --git a/tests/densities/test_measurement_to_density.py b/tests/densities/test_measurement_to_density.py index 440d918..c5d77d7 100644 --- a/tests/densities/test_measurement_to_density.py +++ b/tests/densities/test_measurement_to_density.py @@ -10,6 +10,7 @@ from voxcell import RegionMap # type: ignore import atlas_densities.densities.measurement_to_density as tested +from atlas_densities.exceptions import AtlasDensitiesWarning from tests.densities.test_utils import get_hierarchy, get_hierarchy_info TESTS_PATH = Path(__file__).parent.parent @@ -75,7 +76,9 @@ def test_remove_unknown_regions(region_map, annotations): "source_title": ["Article 1", "Article 2", "Article 1"], } ) - tested.remove_unknown_regions(measurements, region_map, annotations, get_hierarchy_info()) + with pytest.warns(AtlasDensitiesWarning): + tested.remove_unknown_regions(measurements, region_map, annotations, get_hierarchy_info()) + expected = pd.DataFrame( { "brain_region": [ @@ -363,15 +366,17 @@ def get_expected_output(): def test_measurement_to_average_density(): data = get_input_data() expected = get_expected_output() - actual = tested.measurement_to_average_density( - data["region_map"], - data["annotation"], - data["voxel_dimensions"], - data["voxel_volume"], - data["cell_density"], - data["neuron_density"], - data["measurements"], - ) + + with pytest.warns(AtlasDensitiesWarning): + actual = tested.measurement_to_average_density( + data["region_map"], + data["annotation"], + data["voxel_dimensions"], + data["voxel_volume"], + data["cell_density"], + data["neuron_density"], + data["measurements"], + ) pdt.assert_frame_equal(actual, expected) diff --git a/tests/densities/test_mtype_densities_from_composition.py b/tests/densities/test_mtype_densities_from_composition.py index f287e4b..6ac5ad9 100644 --- a/tests/densities/test_mtype_densities_from_composition.py +++ b/tests/densities/test_mtype_densities_from_composition.py @@ -130,12 +130,9 @@ def metadata(): } -@pytest.fixture -def density(annotation): - return np.full_like(annotation.raw, fill_value=0.3, dtype=np.float32) - +def test_create_from_composition(annotation, region_map, metadata, taxonomy, composition): + density = np.full_like(annotation.raw, fill_value=0.3, dtype=np.float32) -def test_create_from_composition(annotation, region_map, metadata, density, taxonomy, composition): per_mtype_density = { mtype: mtype_density for mtype, mtype_density in tested.create_from_composition( @@ -162,7 +159,7 @@ def test_create_from_composition(annotation, region_map, metadata, density, taxo expected_density = np.zeros_like(density) expected_density[0, 0, pos_layer] = (mtype_average_density / layer_sum) * density[ - ..., pos_layer + 0, 0, pos_layer ] mtype_density = per_mtype_density[mtype] diff --git a/tests/densities/test_mtype_densities_from_profiles.py b/tests/densities/test_mtype_densities_from_profiles.py index 9e72b30..a1b8a68 100644 --- a/tests/densities/test_mtype_densities_from_profiles.py +++ b/tests/densities/test_mtype_densities_from_profiles.py @@ -26,11 +26,13 @@ def test_slice_layer(): def create_density_profile_collection(mapping_path="mapping.tsv"): - return tested.DensityProfileCollection.load( - DATA_PATH / "meta" / mapping_path, - DATA_PATH / "meta" / "layers.tsv", - DATA_PATH / "mtypes", - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return tested.DensityProfileCollection.load( + DATA_PATH / "meta" / mapping_path, + DATA_PATH / "meta" / "layers.tsv", + DATA_PATH / "mtypes", + ) def create_slicer_data(): @@ -150,9 +152,12 @@ def expected_profile_data(): def test_density_profile_collection_loading(expected_profile_data): - density_profile_collection = None with warnings.catch_warnings(record=True) as w: - density_profile_collection = create_density_profile_collection() + density_profile_collection = tested.DensityProfileCollection.load( + DATA_PATH / "meta" / "mapping.tsv", + DATA_PATH / "meta" / "layers.tsv", + DATA_PATH / "mtypes", + ) msg = str(w[0].message) assert "No inhibitory cells assigned to slice 0 of layer_3" in msg @@ -165,10 +170,11 @@ def test_density_profile_collection_loading(expected_profile_data): def test_density_profile_collection_loading_exc_only(expected_profile_data): - density_profile_collection = None with warnings.catch_warnings(record=True) as w: - density_profile_collection = create_density_profile_collection( - mapping_path="mapping_exc_only.tsv" + density_profile_collection = tested.DensityProfileCollection.load( + DATA_PATH / "meta" / "mapping_exc_only.tsv", + DATA_PATH / "meta" / "layers.tsv", + DATA_PATH / "mtypes", ) assert not w diff --git a/tests/densities/test_optimization_initialization.py b/tests/densities/test_optimization_initialization.py index f15bdc3..3f54b62 100644 --- a/tests/densities/test_optimization_initialization.py +++ b/tests/densities/test_optimization_initialization.py @@ -10,7 +10,7 @@ import pytest import atlas_densities.densities.inhibitory_neuron_densities_optimization as tested -from atlas_densities.exceptions import AtlasDensitiesError +from atlas_densities.exceptions import AtlasDensitiesError, AtlasDensitiesWarning @pytest.fixture @@ -961,13 +961,14 @@ def expected_aub_bub(): def test_create_aub_and_bub(aub_and_bub_data): - A_ub, b_ub = tested.create_aub_and_bub( - aub_and_bub_data["x_result"], - aub_and_bub_data["region_counts"], - aub_and_bub_data["x_map"], - aub_and_bub_data["deltas_map"], - aub_and_bub_data["hierarchy_info"], - ) + with pytest.warns(AtlasDensitiesWarning): + A_ub, b_ub = tested.create_aub_and_bub( + aub_and_bub_data["x_result"], + aub_and_bub_data["region_counts"], + aub_and_bub_data["x_map"], + aub_and_bub_data["deltas_map"], + aub_and_bub_data["hierarchy_info"], + ) expected_A_ub, expected_b_ub = expected_aub_bub()