Merge branch 'main' into ig/backed_not_implemented

.codecov.yml

@@ -1,16 +1,16 @@
 # Based on pydata/xarray
 codecov:
-  require_ci_to_pass: no
+  require_ci_to_pass: false
 
 coverage:
   status:
     project:
       default:
-        # Require 1% coverage, i.e., always succeed
-        target: 1
+        # Require 75% coverage
+        target: 75
     changes: false
 
 comment:
   layout: "diff, flags, files"
   behavior: once
-  require_base: no
+  require_base: false

docs/release-notes/1.10.2.md

@@ -20,4 +20,6 @@
 
 ```{rubric} Performance
 ```
+
 * `sparse_mean_variance_axis` now uses all cores for the calculations {pr}`3015` {smaller}`S Dicks`
+* Speed up {func}`~scanpy.pp.scrublet` {pr}`3044` {smaller}`S Dicks` and {pr}`3056` {smaller}`P Angerer`

pyproject.toml

@@ -152,6 +152,10 @@ dask-ml = ["dask-ml", "scanpy[dask]"]  # Dask-ML for sklearn-like API
 
 [tool.hatch.build]
 exclude = ["scanpy/tests"]
+[tool.hatch.build.targets.wheel]
+# TODO: move scanpy to src/
+only-include = ["src/testing", "scanpy"]
+sources = ["src", "."]
 [tool.hatch.version]
 source = "vcs"
 [tool.hatch.build.hooks.vcs]
@@ -162,6 +166,7 @@ addopts = [
     "--import-mode=importlib",
     "--strict-markers",
     "--doctest-modules",
+    "-ptesting.scanpy._pytest",
 ]
 testpaths = ["scanpy"]
 norecursedirs = ["scanpy/tests/_images"]
@@ -198,7 +203,7 @@ filterwarnings = [
 [tool.coverage.run]
 data_file = "test-data/coverage"
 source_pkgs = ["scanpy"]
-omit = ["*/tests/*"]
+omit = ["scanpy/tests/*", "src/testing/*"]
 [tool.coverage.xml]
 output = "test-data/coverage.xml"
 [tool.coverage.paths]
@@ -241,7 +246,7 @@ ignore = [
 # Do not assign a lambda expression, use a def
 "scanpy/tools/_rank_genes_groups.py" = ["E731"]
 [tool.ruff.lint.isort]
-known-first-party = ["scanpy"]
+known-first-party = ["scanpy", "testing.scanpy"]
 required-imports = ["from __future__ import annotations"]
 [tool.ruff.lint.flake8-tidy-imports.banned-api]
 "pytest.importorskip".msg = "Use the “@needs” decorator/mark instead"

scanpy/external/pl.py

@@ -10,6 +10,7 @@
 
 from .._compat import old_positionals
 from .._utils import _doc_params
+from .._utils._doctests import doctest_needs
 from ..plotting import _scrublet, _utils, embedding
 from ..plotting._docs import (
     doc_adata_color_etc,
@@ -18,7 +19,6 @@
     doc_show_save_ax,
 )
 from ..plotting._tools.scatterplots import _wraps_plot_scatter
-from ..testing._doctests import doctest_needs
 from .tl._wishbone import _anndata_to_wishbone
 
 if TYPE_CHECKING:

scanpy/external/pp/_bbknn.py

@@ -3,7 +3,7 @@
 from typing import TYPE_CHECKING, Callable
 
 from ..._compat import old_positionals
-from ...testing._doctests import doctest_needs
+from ..._utils._doctests import doctest_needs
 
 if TYPE_CHECKING:
     from anndata import AnnData

scanpy/external/pp/_harmony_integrate.py

@@ -9,7 +9,7 @@
 import numpy as np
 
 from ..._compat import old_positionals
-from ...testing._doctests import doctest_needs
+from ..._utils._doctests import doctest_needs
 
 if TYPE_CHECKING:
     from anndata import AnnData

scanpy/external/pp/_hashsolo.py

@@ -26,7 +26,7 @@
 
 from ..._compat import old_positionals
 from ..._utils import check_nonnegative_integers
-from ...testing._doctests import doctest_skip
+from ..._utils._doctests import doctest_skip
 
 if TYPE_CHECKING:
     from collections.abc import Sequence

scanpy/external/pp/_magic.py

@@ -10,7 +10,7 @@
 
 from ... import logging as logg
 from ..._settings import settings
-from ...testing._doctests import doctest_needs
+from ..._utils._doctests import doctest_needs
 
 if TYPE_CHECKING:
     from collections.abc import Sequence

scanpy/external/pp/_scanorama_integrate.py

@@ -9,7 +9,7 @@
 import numpy as np
 
 from ..._compat import old_positionals
-from ...testing._doctests import doctest_needs
+from ..._utils._doctests import doctest_needs
 
 if TYPE_CHECKING:
     from anndata import AnnData

scanpy/external/tl/_harmony_timeseries.py

@@ -12,7 +12,7 @@
 
 from ... import logging as logg
 from ..._compat import old_positionals
-from ...testing._doctests import doctest_needs
+from ..._utils._doctests import doctest_needs
 
 if TYPE_CHECKING:
     from anndata import AnnData

scanpy/external/tl/_palantir.py

@@ -10,7 +10,7 @@
 
 from ... import logging as logg
 from ..._compat import old_positionals
-from ...testing._doctests import doctest_needs
+from ..._utils._doctests import doctest_needs
 
 if TYPE_CHECKING:
     from anndata import AnnData

scanpy/external/tl/_phate.py

@@ -9,7 +9,7 @@
 from ... import logging as logg
 from ..._compat import old_positionals
 from ..._settings import settings
-from ...testing._doctests import doctest_needs
+from ..._utils._doctests import doctest_needs
 
 if TYPE_CHECKING:
     from anndata import AnnData

scanpy/external/tl/_phenograph.py

@@ -12,7 +12,7 @@
 from ... import logging as logg
 from ..._compat import old_positionals
 from ..._utils import renamed_arg
-from ...testing._doctests import doctest_needs
+from ..._utils._doctests import doctest_needs
 
 if TYPE_CHECKING:
     import numpy as np

scanpy/external/tl/_pypairs.py

@@ -10,7 +10,7 @@
 from packaging.version import Version
 
 from ..._settings import settings
-from ...testing._doctests import doctest_needs
+from ..._utils._doctests import doctest_needs
 
 if TYPE_CHECKING:
     import pandas as pd

scanpy/external/tl/_sam.py

@@ -8,7 +8,7 @@
 
 from ... import logging as logg
 from ..._compat import old_positionals
-from ...testing._doctests import doctest_needs
+from ..._utils._doctests import doctest_needs
 
 if TYPE_CHECKING:
     from anndata import AnnData

scanpy/external/tl/_trimap.py

@@ -11,7 +11,7 @@
 from ... import logging as logg
 from ..._compat import old_positionals
 from ..._settings import settings
-from ...testing._doctests import doctest_needs
+from ..._utils._doctests import doctest_needs
 
 if TYPE_CHECKING:
     from anndata import AnnData

scanpy/external/tl/_wishbone.py

@@ -8,7 +8,7 @@
 
 from ... import logging
 from ..._compat import old_positionals
-from ...testing._doctests import doctest_needs
+from ..._utils._doctests import doctest_needs
 
 if TYPE_CHECKING:
     from collections.abc import Collection, Iterable

scanpy/preprocessing/_scrublet/__init__.py

@@ -199,10 +199,11 @@ def _run_scrublet(ad_obs: AnnData, ad_sim: AnnData | None = None):
             pp.normalize_total(ad_obs)
 
             # HVG process needs log'd data.
-
-            logged = pp.log1p(ad_obs, copy=True)
-            pp.highly_variable_genes(logged)
-            ad_obs = ad_obs[:, logged.var["highly_variable"]].copy()
+            ad_obs.layers["log1p"] = ad_obs.X.copy()
+            pp.log1p(ad_obs, layer="log1p")
+            pp.highly_variable_genes(ad_obs, layer="log1p")
+            del ad_obs.layers["log1p"]
+            ad_obs = ad_obs[:, ad_obs.var["highly_variable"]].copy()
 
             # Simulate the doublets based on the raw expressions from the normalised
             # and filtered object.
@@ -214,7 +215,7 @@ def _run_scrublet(ad_obs: AnnData, ad_sim: AnnData | None = None):
                 synthetic_doublet_umi_subsampling=synthetic_doublet_umi_subsampling,
                 random_seed=random_state,
             )
-
+            del ad_obs.layers["raw"]
             if log_transform:
                 pp.log1p(ad_obs)
                 pp.log1p(ad_sim)

scanpy/preprocessing/_scrublet/pipeline.py

@@ -5,7 +5,9 @@
 import numpy as np
 from scipy import sparse
 
-from .sparse_utils import sparse_multiply, sparse_var, sparse_zscore
+from scanpy.preprocessing._utils import _get_mean_var
+
+from .sparse_utils import sparse_multiply, sparse_zscore
 
 if TYPE_CHECKING:
     from ..._utils import AnyRandom
@@ -20,7 +22,8 @@ def mean_center(self: Scrublet) -> None:
 
 
 def normalize_variance(self: Scrublet) -> None:
-    gene_stdevs = np.sqrt(sparse_var(self._counts_obs_norm, axis=0))
+    _, gene_vars = _get_mean_var(self._counts_obs_norm, axis=0)
+    gene_stdevs = np.sqrt(gene_vars)
     self._counts_obs_norm = sparse_multiply(self._counts_obs_norm.T, 1 / gene_stdevs).T
     if self._counts_sim_norm is not None:
         self._counts_sim_norm = sparse_multiply(
@@ -29,8 +32,8 @@ def normalize_variance(self: Scrublet) -> None:
 
 
 def zscore(self: Scrublet) -> None:
-    gene_means = self._counts_obs_norm.mean(0)
-    gene_stdevs = np.sqrt(sparse_var(self._counts_obs_norm, axis=0))
+    gene_means, gene_vars = _get_mean_var(self._counts_obs_norm, axis=0)
+    gene_stdevs = np.sqrt(gene_vars)
     self._counts_obs_norm = sparse_zscore(
         self._counts_obs_norm, gene_mean=gene_means, gene_stdev=gene_stdevs
     )

scanpy/preprocessing/_scrublet/sparse_utils.py

@@ -1,40 +1,28 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Literal
+from typing import TYPE_CHECKING
 
 import numpy as np
 from scipy import sparse
 
+from scanpy.preprocessing._utils import _get_mean_var
+
 from ..._utils import AnyRandom, get_random_state
 
 if TYPE_CHECKING:
     from numpy.typing import NDArray
 
 
-def sparse_var(
-    E: sparse.csr_matrix | sparse.csc_matrix,
-    *,
-    axis: Literal[0, 1],
-) -> NDArray[np.float64]:
-    """variance across the specified axis"""
-
-    mean_gene: NDArray[np.float64] = E.mean(axis=axis).A.squeeze()
-    tmp: sparse.csc_matrix | sparse.csr_matrix = E.copy()
-    tmp.data **= 2
-    return tmp.mean(axis=axis).A.squeeze() - mean_gene**2
-
-
 def sparse_multiply(
     E: sparse.csr_matrix | sparse.csc_matrix | NDArray[np.float64],
     a: float | int | NDArray[np.float64],
 ) -> sparse.csr_matrix | sparse.csc_matrix:
     """multiply each row of E by a scalar"""
 
     nrow = E.shape[0]
-    w = sparse.lil_matrix((nrow, nrow))
-    w.setdiag(a)
+    w = sparse.dia_matrix((a, 0), shape=(nrow, nrow), dtype=a.dtype)
     r = w @ E
-    if isinstance(r, (np.matrix, np.ndarray)):
+    if isinstance(r, np.ndarray):
         return sparse.csc_matrix(r)
     return r
 
@@ -46,11 +34,9 @@ def sparse_zscore(
     gene_stdev: NDArray[np.float64] | None = None,
 ) -> sparse.csr_matrix | sparse.csc_matrix:
     """z-score normalize each column of E"""
-
-    if gene_mean is None:
-        gene_mean = E.mean(0)
-    if gene_stdev is None:
-        gene_stdev = np.sqrt(sparse_var(E, axis=0))
+    if gene_mean is None or gene_stdev is None:
+        gene_means, gene_stdevs = _get_mean_var(E, axis=0)
+        gene_stdevs = np.sqrt(gene_stdevs)
     return sparse_multiply(np.asarray((E - gene_mean).T), 1 / gene_stdev).T
 
 

scanpy/queries/_queries.py

@@ -8,8 +8,8 @@
 from anndata import AnnData
 
 from .._utils import _doc_params
+from .._utils._doctests import doctest_needs
 from ..get import rank_genes_groups_df
-from ..testing._doctests import doctest_needs
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Mapping

scanpy/tests/conftest.py

@@ -7,16 +7,14 @@
 
 import pytest
 
-pytest_plugins = ["scanpy.testing._pytest"]
-
 # just import for the IMPORTED check
 import scanpy as _sc  # noqa: F401
 
 if TYPE_CHECKING:  # So editors understand that we’re using those fixtures
     import os
     from collections.abc import Generator
 
-    from scanpy.testing._pytest.fixtures import *  # noqa: F403
+    from testing.scanpy._pytest.fixtures import *  # noqa: F403
 
 # define this after importing scanpy but before running tests
 IMPORTED = frozenset(sys.modules.keys())

scanpy/tests/external/test_harmony_integrate.py

@@ -2,8 +2,8 @@
 
 import scanpy as sc
 import scanpy.external as sce
-from scanpy.testing._helpers.data import pbmc3k
-from scanpy.testing._pytest.marks import needs
+from testing.scanpy._helpers.data import pbmc3k
+from testing.scanpy._pytest.marks import needs
 
 pytestmark = [needs.harmonypy]
 

scanpy/tests/external/test_harmony_timeseries.py

@@ -6,8 +6,8 @@
 
 import scanpy as sc
 import scanpy.external as sce
-from scanpy.testing._helpers.data import pbmc3k
-from scanpy.testing._pytest.marks import needs
+from testing.scanpy._helpers.data import pbmc3k
+from testing.scanpy._pytest.marks import needs
 
 pytestmark = [needs.harmony]
 

scanpy/tests/external/test_magic.py

@@ -4,7 +4,7 @@
 from anndata import AnnData
 
 import scanpy as sc
-from scanpy.testing._pytest.marks import needs
+from testing.scanpy._pytest.marks import needs
 
 pytestmark = [needs.magic]
 

scanpy/tests/external/test_palantir.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 import scanpy.external as sce
-from scanpy.testing._helpers.data import pbmc3k_processed
-from scanpy.testing._pytest.marks import needs
+from testing.scanpy._helpers.data import pbmc3k_processed
+from testing.scanpy._pytest.marks import needs
 
 pytestmark = [needs.palantir]
 

scanpy/tests/external/test_phenograph.py

@@ -6,7 +6,7 @@
 
 import scanpy as sc
 import scanpy.external as sce
-from scanpy.testing._pytest.marks import needs
+from testing.scanpy._pytest.marks import needs
 
 pytestmark = [needs.phenograph]