Merge remote-tracking branch 'origin/main' into fix_nan_id_counts

atlas_densities/app/mtype_densities.py

@@ -408,7 +408,7 @@ def create_from_composition(
 
 def _load_neuronal_mtype_taxonomy(filename: str) -> pd.DataFrame:
     """Loads the taxonomy tsv file into a dataframe"""
-    return pd.read_csv(filename, header=0, delim_whitespace=True)
+    return pd.read_csv(filename, header=0, sep=r"\s+")
 
 
 def _validate_mtype_taxonomy(taxonomy: pd.DataFrame) -> None:

atlas_densities/combination/markers_combinator.py

@@ -127,14 +127,21 @@ def compute_voxel_count(
     # 'A Cell Atlas for the Mouse Brain' in that we use the weights 1.0 / E_marker normalized by
     # their sums instead of the coefficients C_marker = 1.0 / (E_marker * N_marker).
     combination_data.sort_index(inplace=True)
-    intensities = combination_data.groupby("cellType").apply(
-        lambda x: np.average(
+
+    def func(x):
+        return np.average(
             np.array(list(x.volume)),
-            weights=1.0 / np.array(list(x["averageExpressionIntensity"])),
+            weights=1.0 / x["averageExpressionIntensity"].to_numpy(),
             axis=0,
         )
+
+    kwargs = {}
+    if tuple(pd.__version__.split(".")) > ("2", "2", "0"):
+        kwargs["include_groups"] = False
+
+    intensities = (
+        combination_data.groupby("cellType").apply(func, **kwargs).rename("intensity").to_frame()
     )
-    intensities = pd.DataFrame(intensities, columns=["intensity"])
     intensities["proportion"] = proportions
 
     # Average overall glia intensity

atlas_densities/densities/excel_reader.py

@@ -164,7 +164,7 @@ def compute_kim_et_al_neuron_densities(
         engine="openpyxl",
     ).set_index("ROI")
     for acronym in kim_et_al_mmc3.index:
-        full_name = kim_et_al_mmc3.loc[acronym][0]
+        full_name = kim_et_al_mmc3.loc[acronym, "full_name"]
         if not isinstance(full_name, str) or not full_name:
             warn(
                 f"Region with acronym {acronym} has no valid full name. "
@@ -524,7 +524,7 @@ def read_homogenous_neuron_type_regions(measurements_path: str | "Path") -> pd.D
     )
     excitatory_mask = homogenous_regions_worksheet["comment"] == "Purely excitatory region"
     homogenous_regions_worksheet["cell_type"] = "inhibitory"
-    homogenous_regions_worksheet["cell_type"][excitatory_mask] = "excitatory"
+    homogenous_regions_worksheet.loc[excitatory_mask, "cell_type"] = "excitatory"
     homogenous_regions_worksheet.drop(columns=["comment"], inplace=True)
 
     return homogenous_regions_worksheet

atlas_densities/densities/measurement_to_density.py

@@ -277,7 +277,6 @@ def remove_unknown_regions(
         hierarchy_info: data frame returned by
             :func:`atlas_densities.densities.utils.get_hierarchy_info`.
     """
-    pd.set_option("display.max_colwidth", None)
     indices_ids = measurements.index[
         ~measurements["brain_region"].isin(hierarchy_info["brain_region"])
     ]

atlas_densities/densities/utils.py

@@ -530,7 +530,7 @@ def compute_region_volumes(
     )
 
     ids, counts = np.unique(annotation, return_counts=True)
-    result["id_volume"].update(pd.Series(counts * voxel_volume, index=ids))
+    result.update({"id_volume": pd.Series(counts * voxel_volume, index=ids)})
 
     volumes = []
     for id_ in hierarchy_info.index:

pyproject.toml

@@ -0,0 +1,9 @@
+[tool.pytest.ini_options]
+addopts = [
+    "--import-mode=importlib",
+]
+filterwarnings = [
+    "ignore::DeprecationWarning:nptyping",
+    "ignore::UserWarning:openpyxl",
+]
+

tests/app/test_cell_densities.py

@@ -1,6 +1,7 @@
 """test cell_densities"""
 
 import json
+import warnings
 from pathlib import Path
 
 import numpy as np
@@ -253,7 +254,9 @@ def _get_measurements_to_average_densities_result(runner, hierarchy_path, measur
         # fmt: on
     ]
 
-    return runner.invoke(tested.app, args)
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        return runner.invoke(tested.app, args)
 
 
 def test_measurements_to_average_densities():

tests/app/test_mtype_densities.py

@@ -1,6 +1,7 @@
 """test app/mtype_densities"""
 import json
 import os
+import warnings
 from copy import deepcopy
 from pathlib import Path
 
@@ -27,21 +28,23 @@
 
 
 def get_result_from_profiles(runner, td):
-    return runner.invoke(
-        tested.app,
-        [
-            # fmt: off
-            "--log-output-path", td,
-            "create-from-profile",
-            "--annotation-path", "annotation.nrrd",
-            "--hierarchy-path", "hierarchy.json",
-            "--metadata-path", "metadata.json",
-            "--direction-vectors-path", "direction_vectors.nrrd",
-            "--mtypes-config-path", "config.yaml",
-            "--output-dir", "output_dir",
-            # fmt: on
-        ],
-    )
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        return runner.invoke(
+            tested.app,
+            [
+                # fmt: off
+                "--log-output-path", td,
+                "create-from-profile",
+                "--annotation-path", "annotation.nrrd",
+                "--hierarchy-path", "hierarchy.json",
+                "--metadata-path", "metadata.json",
+                "--direction-vectors-path", "direction_vectors.nrrd",
+                "--mtypes-config-path", "config.yaml",
+                "--output-dir", "output_dir",
+                # fmt: on
+            ],
+        )
 
 
 def test_mtype_densities_from_profiles(tmp_path):

tests/densities/test_excel_reader.py

@@ -178,10 +178,11 @@ def check_non_negative_values(dataframe):
 
 
 def test_read_kim_et_al_neuron_densities():
-    warnings.simplefilter("ignore")
-    dataframe = tested.read_kim_et_al_neuron_densities(
-        REGION_MAP, Path(MEASUREMENTS_PATH, "mmc3.xlsx")
-    )
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        dataframe = tested.read_kim_et_al_neuron_densities(
+            REGION_MAP, Path(MEASUREMENTS_PATH, "mmc3.xlsx")
+        )
 
     check_column_names(dataframe)
     check_column_types(dataframe)
@@ -331,13 +332,14 @@ def test_read_non_density_measurements():
 
 
 def test_read_measurements():
-    warnings.simplefilter("ignore")
-    dataframe = tested.read_measurements(
-        REGION_MAP,
-        Path(MEASUREMENTS_PATH, "mmc3.xlsx"),
-        Path(MEASUREMENTS_PATH, "gaba_papers.xlsx"),
-        Path(MEASUREMENTS_PATH, "non_density_measurements.csv"),
-    )
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        dataframe = tested.read_measurements(
+            REGION_MAP,
+            Path(MEASUREMENTS_PATH, "mmc3.xlsx"),
+            Path(MEASUREMENTS_PATH, "gaba_papers.xlsx"),
+            Path(MEASUREMENTS_PATH, "non_density_measurements.csv"),
+        )
     check_column_names(dataframe)
     check_non_negative_values(dataframe)
     check_columns_na(dataframe)
@@ -357,3 +359,10 @@ def test_read_measurements():
     # TODO: one duplicate title because of a trailing endline character
     assert len(set(dataframe["source_title"])) == 48
     assert len(set(dataframe["comment"])) >= 48
+
+
+def test_read_homogenous_neuron_type_regions():
+    res = tested.read_homogenous_neuron_type_regions(MEASUREMENTS_PATH / "gaba_papers.xlsx")
+    assert len(res) == 60
+    assert np.count_nonzero(res.cell_type == "excitatory") == 3
+    assert np.count_nonzero(res.cell_type == "inhibitory") == 57

tests/densities/test_inhibitory_neuron_density_optimization.py

@@ -10,7 +10,7 @@
 import pytest
 
 import atlas_densities.densities.inhibitory_neuron_densities_optimization as tested
-from atlas_densities.exceptions import AtlasDensitiesError
+from atlas_densities.exceptions import AtlasDensitiesError, AtlasDensitiesWarning
 
 
 def test_resize_average_densities():
@@ -322,13 +322,15 @@ def get_initialization_data_4():
 
 def test_create_inhibitory_neuron_densities_4():
     data = get_initialization_data_4()
-    densities = tested.create_inhibitory_neuron_densities(
-        data["hierarchy"],
-        data["annotation"],
-        data["voxel_volume"],
-        data["neuron_density"],
-        data["average_densities"],
-    )
+
+    with pytest.warns(AtlasDensitiesWarning):
+        densities = tested.create_inhibitory_neuron_densities(
+            data["hierarchy"],
+            data["annotation"],
+            data["voxel_volume"],
+            data["neuron_density"],
+            data["average_densities"],
+        )
 
     expected = {
         "gad67+": np.array([[[0.5, 0.5, 1.0, 1.0, 1.5, 1.5]]]),

tests/densities/test_measurement_to_density.py

@@ -10,6 +10,7 @@
 from voxcell import RegionMap  # type: ignore
 
 import atlas_densities.densities.measurement_to_density as tested
+from atlas_densities.exceptions import AtlasDensitiesWarning
 from tests.densities.test_utils import get_hierarchy, get_hierarchy_info
 
 TESTS_PATH = Path(__file__).parent.parent
@@ -75,7 +76,9 @@ def test_remove_unknown_regions(region_map, annotations):
             "source_title": ["Article 1", "Article 2", "Article 1"],
         }
     )
-    tested.remove_unknown_regions(measurements, region_map, annotations, get_hierarchy_info())
+    with pytest.warns(AtlasDensitiesWarning):
+        tested.remove_unknown_regions(measurements, region_map, annotations, get_hierarchy_info())
+
     expected = pd.DataFrame(
         {
             "brain_region": [
@@ -363,15 +366,17 @@ def get_expected_output():
 def test_measurement_to_average_density():
     data = get_input_data()
     expected = get_expected_output()
-    actual = tested.measurement_to_average_density(
-        data["region_map"],
-        data["annotation"],
-        data["voxel_dimensions"],
-        data["voxel_volume"],
-        data["cell_density"],
-        data["neuron_density"],
-        data["measurements"],
-    )
+
+    with pytest.warns(AtlasDensitiesWarning):
+        actual = tested.measurement_to_average_density(
+            data["region_map"],
+            data["annotation"],
+            data["voxel_dimensions"],
+            data["voxel_volume"],
+            data["cell_density"],
+            data["neuron_density"],
+            data["measurements"],
+        )
     pdt.assert_frame_equal(actual, expected)
 
 

tests/densities/test_mtype_densities_from_composition.py

@@ -130,12 +130,9 @@ def metadata():
     }
 
 
-@pytest.fixture
-def density(annotation):
-    return np.full_like(annotation.raw, fill_value=0.3, dtype=np.float32)
-
+def test_create_from_composition(annotation, region_map, metadata, taxonomy, composition):
+    density = np.full_like(annotation.raw, fill_value=0.3, dtype=np.float32)
 
-def test_create_from_composition(annotation, region_map, metadata, density, taxonomy, composition):
     per_mtype_density = {
         mtype: mtype_density
         for mtype, mtype_density in tested.create_from_composition(
@@ -162,7 +159,7 @@ def test_create_from_composition(annotation, region_map, metadata, density, taxo
 
         expected_density = np.zeros_like(density)
         expected_density[0, 0, pos_layer] = (mtype_average_density / layer_sum) * density[
-            ..., pos_layer
+            0, 0, pos_layer
         ]
 
         mtype_density = per_mtype_density[mtype]

tests/densities/test_mtype_densities_from_profiles.py

@@ -26,11 +26,13 @@ def test_slice_layer():
 
 
 def create_density_profile_collection(mapping_path="mapping.tsv"):
-    return tested.DensityProfileCollection.load(
-        DATA_PATH / "meta" / mapping_path,
-        DATA_PATH / "meta" / "layers.tsv",
-        DATA_PATH / "mtypes",
-    )
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        return tested.DensityProfileCollection.load(
+            DATA_PATH / "meta" / mapping_path,
+            DATA_PATH / "meta" / "layers.tsv",
+            DATA_PATH / "mtypes",
+        )
 
 
 def create_slicer_data():
@@ -150,9 +152,12 @@ def expected_profile_data():
 
 
 def test_density_profile_collection_loading(expected_profile_data):
-    density_profile_collection = None
     with warnings.catch_warnings(record=True) as w:
-        density_profile_collection = create_density_profile_collection()
+        density_profile_collection = tested.DensityProfileCollection.load(
+            DATA_PATH / "meta" / "mapping.tsv",
+            DATA_PATH / "meta" / "layers.tsv",
+            DATA_PATH / "mtypes",
+        )
         msg = str(w[0].message)
         assert "No inhibitory cells assigned to slice 0 of layer_3" in msg
 
@@ -165,10 +170,11 @@ def test_density_profile_collection_loading(expected_profile_data):
 
 
 def test_density_profile_collection_loading_exc_only(expected_profile_data):
-    density_profile_collection = None
     with warnings.catch_warnings(record=True) as w:
-        density_profile_collection = create_density_profile_collection(
-            mapping_path="mapping_exc_only.tsv"
+        density_profile_collection = tested.DensityProfileCollection.load(
+            DATA_PATH / "meta" / "mapping_exc_only.tsv",
+            DATA_PATH / "meta" / "layers.tsv",
+            DATA_PATH / "mtypes",
         )
         assert not w
 

tests/densities/test_optimization_initialization.py

@@ -10,7 +10,7 @@
 import pytest
 
 import atlas_densities.densities.inhibitory_neuron_densities_optimization as tested
-from atlas_densities.exceptions import AtlasDensitiesError
+from atlas_densities.exceptions import AtlasDensitiesError, AtlasDensitiesWarning
 
 
 @pytest.fixture
@@ -961,13 +961,14 @@ def expected_aub_bub():
 
 
 def test_create_aub_and_bub(aub_and_bub_data):
-    A_ub, b_ub = tested.create_aub_and_bub(
-        aub_and_bub_data["x_result"],
-        aub_and_bub_data["region_counts"],
-        aub_and_bub_data["x_map"],
-        aub_and_bub_data["deltas_map"],
-        aub_and_bub_data["hierarchy_info"],
-    )
+    with pytest.warns(AtlasDensitiesWarning):
+        A_ub, b_ub = tested.create_aub_and_bub(
+            aub_and_bub_data["x_result"],
+            aub_and_bub_data["region_counts"],
+            aub_and_bub_data["x_map"],
+            aub_and_bub_data["deltas_map"],
+            aub_and_bub_data["hierarchy_info"],
+        )
 
     expected_A_ub, expected_b_ub = expected_aub_bub()