From c1409229dbc93fdc083f9fbfddaee597307d9d23 Mon Sep 17 00:00:00 2001 From: pattonw Date: Wed, 14 Feb 2024 13:18:18 -0800 Subject: [PATCH 01/15] use kaiming initialization for conv layers --- dacapo/experiments/model.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/dacapo/experiments/model.py b/dacapo/experiments/model.py index 8ca2b2b9e..75777cd81 100644 --- a/dacapo/experiments/model.py +++ b/dacapo/experiments/model.py @@ -40,6 +40,13 @@ def __init__( ) self.eval_activation = eval_activation + # UPDATE WEIGHT INITIALIZATION TO USE KAIMING + # TODO: put this somewhere better, there might be + # conv layers that aren't follwed by relus? + for _name, layer in self.named_modules(): + if isinstance(layer, torch.nn.modules.conv._ConvNd): + torch.nn.init.kaiming_normal_(layer.weight, nonlinearity="relu") + def forward(self, x): result = self.chain(x) if not self.training and self.eval_activation is not None: From b8f6e50aac4d73302c29d1c0d13d6fa997df0b39 Mon Sep 17 00:00:00 2001 From: pattonw Date: Wed, 14 Feb 2024 13:22:42 -0800 Subject: [PATCH 02/15] crop_array bugfix handle case where crop_array roi is larger than source arrays roi --- dacapo/experiments/datasplits/datasets/arrays/crop_array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dacapo/experiments/datasplits/datasets/arrays/crop_array.py b/dacapo/experiments/datasplits/datasets/arrays/crop_array.py index 1782f028e..04b163513 100644 --- a/dacapo/experiments/datasplits/datasets/arrays/crop_array.py +++ b/dacapo/experiments/datasplits/datasets/arrays/crop_array.py @@ -35,7 +35,7 @@ def voxel_size(self) -> Coordinate: @property def roi(self) -> Roi: - return self.crop_roi + return self.crop_roi.intersect(self._source_array.roi) @property def writable(self) -> bool: From bc756256a57b2aa2a320b18019af90a6dd71c438 Mon Sep 17 00:00:00 2001 From: pattonw Date: Wed, 14 Feb 2024 13:35:32 -0800 Subject: [PATCH 03/15] handle broken symlink if the file best_weights points to got deleted, then `best_weights.exists()` is false. In this case the `symlink_to` call will fail because the symlink exists, so we need to remove it --- dacapo/store/local_weights_store.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dacapo/store/local_weights_store.py b/dacapo/store/local_weights_store.py index 0935c945b..392285875 100644 --- a/dacapo/store/local_weights_store.py +++ b/dacapo/store/local_weights_store.py @@ -96,7 +96,12 @@ def store_best(self, run: str, iteration: int, dataset: str, criterion: str): if best_weights.exists(): best_weights.unlink() - best_weights.symlink_to(iteration_weights) + try: + best_weights.symlink_to(iteration_weights) + except FileExistsError: + best_weights.unlink() + best_weights.symlink_to(iteration_weights) + with best_weights_json.open("w") as f: f.write(json.dumps({"iteration": iteration})) From 8da6be122d9f0987e942773b3d66978910dad05b Mon Sep 17 00:00:00 2001 From: Jeff Rhoades <37990507+rhoadesScholar@users.noreply.github.com> Date: Wed, 14 Feb 2024 19:06:50 -0500 Subject: [PATCH 04/15] Update pyproject.toml add tool dependency --- pyproject.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1b9d716a2..2697d98e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,9 @@ dependencies = [ "xarray", "cattrs", "numpy-indexed", - "click",] + "click", + "toml", + ] # extras # https://peps.python.org/pep-0621/#dependencies-optional-dependencies @@ -165,4 +167,4 @@ ignore = [ ".pre-commit-config.yaml", ".ruff_cache/**/*", "tests/**/*", -] \ No newline at end of file +] From 74e2e395c99131095030f90962b4db4a747d61b7 Mon Sep 17 00:00:00 2001 From: Diane Adjavon Date: Thu, 15 Feb 2024 10:49:55 -0500 Subject: [PATCH 05/15] fix: Remove circular imports and add missing import --- dacapo/apply.py | 4 ++-- dacapo/blockwise/argmax_worker.py | 4 ++-- dacapo/blockwise/predict_worker.py | 4 ++-- dacapo/blockwise/threshold_worker.py | 4 ++-- .../tasks/post_processors/argmax_post_processor.py | 1 - .../tasks/post_processors/watershed_post_processor.py | 3 +-- dacapo/predict.py | 1 + dacapo/store/array_store.py | 2 +- dacapo/store/local_array_store.py | 2 +- 9 files changed, 12 insertions(+), 13 deletions(-) diff --git a/dacapo/apply.py b/dacapo/apply.py index a701d9272..89cf1120c 100644 --- a/dacapo/apply.py +++ b/dacapo/apply.py @@ -174,8 +174,8 @@ def apply_run( run: Run, parameters: PostProcessorParameters, input_array: Array, - prediction_array_identifier: LocalArrayIdentifier, - output_array_identifier: LocalArrayIdentifier, + prediction_array_identifier: 'LocalArrayIdentifier', + output_array_identifier: 'LocalArrayIdentifier', roi: Optional[Roi] = None, num_cpu_workers: int = 30, output_dtype: Optional[np.dtype] = np.uint8, # type: ignore diff --git a/dacapo/blockwise/argmax_worker.py b/dacapo/blockwise/argmax_worker.py index 22bb825d3..efecca17f 100644 --- a/dacapo/blockwise/argmax_worker.py +++ b/dacapo/blockwise/argmax_worker.py @@ -72,8 +72,8 @@ def start_worker( def spawn_worker( - input_array_identifier: LocalArrayIdentifier, - output_array_identifier: LocalArrayIdentifier, + input_array_identifier: 'LocalArrayIdentifier', + output_array_identifier: 'LocalArrayIdentifier', compute_context: ComputeContext = LocalTorch(), ): """Spawn a worker to predict on a given dataset. diff --git a/dacapo/blockwise/predict_worker.py b/dacapo/blockwise/predict_worker.py index 237062f7e..2866655a0 100644 --- a/dacapo/blockwise/predict_worker.py +++ b/dacapo/blockwise/predict_worker.py @@ -170,8 +170,8 @@ def start_worker( def spawn_worker( run_name: str, iteration: int, - raw_array_identifier: LocalArrayIdentifier, - prediction_array_identifier: LocalArrayIdentifier, + raw_array_identifier: 'LocalArrayIdentifier', + prediction_array_identifier: 'LocalArrayIdentifier', compute_context: ComputeContext = LocalTorch(), ): """Spawn a worker to predict on a given dataset. diff --git a/dacapo/blockwise/threshold_worker.py b/dacapo/blockwise/threshold_worker.py index 2f7af5236..acd2093f3 100644 --- a/dacapo/blockwise/threshold_worker.py +++ b/dacapo/blockwise/threshold_worker.py @@ -73,8 +73,8 @@ def start_worker( def spawn_worker( - input_array_identifier: LocalArrayIdentifier, - output_array_identifier: LocalArrayIdentifier, + input_array_identifier: 'LocalArrayIdentifier', + output_array_identifier: 'LocalArrayIdentifier', threshold: float = 0.0, compute_context: ComputeContext = LocalTorch(), ): diff --git a/dacapo/experiments/tasks/post_processors/argmax_post_processor.py b/dacapo/experiments/tasks/post_processors/argmax_post_processor.py index a3af2d62c..d5b2361da 100644 --- a/dacapo/experiments/tasks/post_processors/argmax_post_processor.py +++ b/dacapo/experiments/tasks/post_processors/argmax_post_processor.py @@ -2,7 +2,6 @@ from dacapo.blockwise.scheduler import run_blockwise from dacapo.compute_context import ComputeContext, LocalTorch from dacapo.experiments.datasplits.datasets.arrays.zarr_array import ZarrArray -from dacapo.store.array_store import LocalArrayIdentifier from .argmax_post_processor_parameters import ArgmaxPostProcessorParameters from .post_processor import PostProcessor import numpy as np diff --git a/dacapo/experiments/tasks/post_processors/watershed_post_processor.py b/dacapo/experiments/tasks/post_processors/watershed_post_processor.py index 99bba201f..bc1ce6c65 100644 --- a/dacapo/experiments/tasks/post_processors/watershed_post_processor.py +++ b/dacapo/experiments/tasks/post_processors/watershed_post_processor.py @@ -1,5 +1,4 @@ from dacapo.experiments.datasplits.datasets.arrays import ZarrArray -from dacapo.store.array_store import LocalArrayIdentifier from .watershed_post_processor_parameters import WatershedPostProcessorParameters from .post_processor import PostProcessor @@ -36,7 +35,7 @@ def set_prediction(self, prediction_array_identifier): def process( self, parameters: WatershedPostProcessorParameters, - output_array_identifier: LocalArrayIdentifier, + output_array_identifier: 'LocalArrayIdentifier', ): output_array = ZarrArray.create_from_array_identifier( output_array_identifier, diff --git a/dacapo/predict.py b/dacapo/predict.py index 131820572..3e5dea79a 100644 --- a/dacapo/predict.py +++ b/dacapo/predict.py @@ -10,6 +10,7 @@ from dacapo.store.local_array_store import LocalArrayIdentifier from dacapo.compute_context import LocalTorch, ComputeContext from dacapo.experiments.datasplits.datasets.arrays import ZarrArray, Array +from dacapo.cli import cli from funlib.geometry import Coordinate, Roi import gunpowder as gp diff --git a/dacapo/store/array_store.py b/dacapo/store/array_store.py index 065196066..7fdc21adc 100644 --- a/dacapo/store/array_store.py +++ b/dacapo/store/array_store.py @@ -62,7 +62,7 @@ def validation_input_arrays( pass @abstractmethod - def remove(self, array_identifier: LocalArrayIdentifier) -> None: + def remove(self, array_identifier: 'LocalArrayIdentifier') -> None: """Remove an array by its identifier.""" pass diff --git a/dacapo/store/local_array_store.py b/dacapo/store/local_array_store.py index c1581fc7b..21d3da0ab 100644 --- a/dacapo/store/local_array_store.py +++ b/dacapo/store/local_array_store.py @@ -85,7 +85,7 @@ def validation_container(self, run_name: str) -> LocalContainerIdentifier: Path(self.__get_run_dir(run_name), "validation.zarr") ) - def remove(self, array_identifier: LocalArrayIdentifier) -> None: + def remove(self, array_identifier: 'LocalArrayIdentifier') -> None: container = array_identifier.container dataset = array_identifier.dataset From 2f65cd6ab288bd58c97349b7fd18e4d33f6637b1 Mon Sep 17 00:00:00 2001 From: adjavon Date: Thu, 15 Feb 2024 15:58:51 +0000 Subject: [PATCH 06/15] :art: Format Python code with psf/black --- dacapo/apply.py | 4 ++-- dacapo/blockwise/argmax_worker.py | 4 ++-- dacapo/blockwise/predict_worker.py | 4 ++-- dacapo/blockwise/threshold_worker.py | 4 ++-- .../tasks/post_processors/watershed_post_processor.py | 2 +- dacapo/store/array_store.py | 2 +- dacapo/store/local_array_store.py | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/dacapo/apply.py b/dacapo/apply.py index 89cf1120c..d4fefb995 100644 --- a/dacapo/apply.py +++ b/dacapo/apply.py @@ -174,8 +174,8 @@ def apply_run( run: Run, parameters: PostProcessorParameters, input_array: Array, - prediction_array_identifier: 'LocalArrayIdentifier', - output_array_identifier: 'LocalArrayIdentifier', + prediction_array_identifier: "LocalArrayIdentifier", + output_array_identifier: "LocalArrayIdentifier", roi: Optional[Roi] = None, num_cpu_workers: int = 30, output_dtype: Optional[np.dtype] = np.uint8, # type: ignore diff --git a/dacapo/blockwise/argmax_worker.py b/dacapo/blockwise/argmax_worker.py index efecca17f..a4e23578a 100644 --- a/dacapo/blockwise/argmax_worker.py +++ b/dacapo/blockwise/argmax_worker.py @@ -72,8 +72,8 @@ def start_worker( def spawn_worker( - input_array_identifier: 'LocalArrayIdentifier', - output_array_identifier: 'LocalArrayIdentifier', + input_array_identifier: "LocalArrayIdentifier", + output_array_identifier: "LocalArrayIdentifier", compute_context: ComputeContext = LocalTorch(), ): """Spawn a worker to predict on a given dataset. diff --git a/dacapo/blockwise/predict_worker.py b/dacapo/blockwise/predict_worker.py index 2866655a0..d4df66ad9 100644 --- a/dacapo/blockwise/predict_worker.py +++ b/dacapo/blockwise/predict_worker.py @@ -170,8 +170,8 @@ def start_worker( def spawn_worker( run_name: str, iteration: int, - raw_array_identifier: 'LocalArrayIdentifier', - prediction_array_identifier: 'LocalArrayIdentifier', + raw_array_identifier: "LocalArrayIdentifier", + prediction_array_identifier: "LocalArrayIdentifier", compute_context: ComputeContext = LocalTorch(), ): """Spawn a worker to predict on a given dataset. diff --git a/dacapo/blockwise/threshold_worker.py b/dacapo/blockwise/threshold_worker.py index acd2093f3..929eebbf0 100644 --- a/dacapo/blockwise/threshold_worker.py +++ b/dacapo/blockwise/threshold_worker.py @@ -73,8 +73,8 @@ def start_worker( def spawn_worker( - input_array_identifier: 'LocalArrayIdentifier', - output_array_identifier: 'LocalArrayIdentifier', + input_array_identifier: "LocalArrayIdentifier", + output_array_identifier: "LocalArrayIdentifier", threshold: float = 0.0, compute_context: ComputeContext = LocalTorch(), ): diff --git a/dacapo/experiments/tasks/post_processors/watershed_post_processor.py b/dacapo/experiments/tasks/post_processors/watershed_post_processor.py index bc1ce6c65..c447694ed 100644 --- a/dacapo/experiments/tasks/post_processors/watershed_post_processor.py +++ b/dacapo/experiments/tasks/post_processors/watershed_post_processor.py @@ -35,7 +35,7 @@ def set_prediction(self, prediction_array_identifier): def process( self, parameters: WatershedPostProcessorParameters, - output_array_identifier: 'LocalArrayIdentifier', + output_array_identifier: "LocalArrayIdentifier", ): output_array = ZarrArray.create_from_array_identifier( output_array_identifier, diff --git a/dacapo/store/array_store.py b/dacapo/store/array_store.py index 7fdc21adc..7c44ab7ab 100644 --- a/dacapo/store/array_store.py +++ b/dacapo/store/array_store.py @@ -62,7 +62,7 @@ def validation_input_arrays( pass @abstractmethod - def remove(self, array_identifier: 'LocalArrayIdentifier') -> None: + def remove(self, array_identifier: "LocalArrayIdentifier") -> None: """Remove an array by its identifier.""" pass diff --git a/dacapo/store/local_array_store.py b/dacapo/store/local_array_store.py index 21d3da0ab..73994d980 100644 --- a/dacapo/store/local_array_store.py +++ b/dacapo/store/local_array_store.py @@ -85,7 +85,7 @@ def validation_container(self, run_name: str) -> LocalContainerIdentifier: Path(self.__get_run_dir(run_name), "validation.zarr") ) - def remove(self, array_identifier: 'LocalArrayIdentifier') -> None: + def remove(self, array_identifier: "LocalArrayIdentifier") -> None: container = array_identifier.container dataset = array_identifier.dataset From 5521843f7b0c70c53f2861bc93b311292d3b1a40 Mon Sep 17 00:00:00 2001 From: Diane Adjavon Date: Thu, 15 Feb 2024 11:26:24 -0500 Subject: [PATCH 07/15] fix: Update the changes for the circular imports One of the files was using `LocalArrayIdentifier`, so in order to remove the circular import entirely we had to empty `store.__init__`. This means that from this point on, all of the `create_*_store` calls need to refer directly to `dacapo.store.create_store` --- dacapo/apply.py | 6 +++--- dacapo/blockwise/argmax_worker.py | 4 ++-- dacapo/blockwise/predict_worker.py | 4 ++-- dacapo/blockwise/threshold_worker.py | 4 ++-- .../tasks/post_processors/argmax_post_processor.py | 1 + .../post_processors/watershed_post_processor.py | 2 +- dacapo/predict.py | 3 +-- dacapo/store/__init__.py | 7 ------- dacapo/store/array_store.py | 2 +- dacapo/store/local_array_store.py | 2 +- dacapo/train.py | 12 ++++++++---- dacapo/validate.py | 2 +- tests/components/test_arrays.py | 2 +- tests/components/test_trainers.py | 2 +- tests/operations/test_apply.py | 2 +- tests/operations/test_train.py | 2 +- tests/operations/test_validate.py | 2 +- 17 files changed, 28 insertions(+), 31 deletions(-) diff --git a/dacapo/apply.py b/dacapo/apply.py index 89cf1120c..7ee0473fb 100644 --- a/dacapo/apply.py +++ b/dacapo/apply.py @@ -14,7 +14,7 @@ from dacapo.predict import predict from dacapo.compute_context import LocalTorch, ComputeContext from dacapo.experiments.datasplits.datasets.arrays import ZarrArray -from dacapo.store import ( +from dacapo.store.create_store import ( create_config_store, create_weights_store, ) @@ -174,8 +174,8 @@ def apply_run( run: Run, parameters: PostProcessorParameters, input_array: Array, - prediction_array_identifier: 'LocalArrayIdentifier', - output_array_identifier: 'LocalArrayIdentifier', + prediction_array_identifier: "LocalArrayIdentifier", + output_array_identifier: "LocalArrayIdentifier", roi: Optional[Roi] = None, num_cpu_workers: int = 30, output_dtype: Optional[np.dtype] = np.uint8, # type: ignore diff --git a/dacapo/blockwise/argmax_worker.py b/dacapo/blockwise/argmax_worker.py index efecca17f..a4e23578a 100644 --- a/dacapo/blockwise/argmax_worker.py +++ b/dacapo/blockwise/argmax_worker.py @@ -72,8 +72,8 @@ def start_worker( def spawn_worker( - input_array_identifier: 'LocalArrayIdentifier', - output_array_identifier: 'LocalArrayIdentifier', + input_array_identifier: "LocalArrayIdentifier", + output_array_identifier: "LocalArrayIdentifier", compute_context: ComputeContext = LocalTorch(), ): """Spawn a worker to predict on a given dataset. diff --git a/dacapo/blockwise/predict_worker.py b/dacapo/blockwise/predict_worker.py index 2866655a0..d4df66ad9 100644 --- a/dacapo/blockwise/predict_worker.py +++ b/dacapo/blockwise/predict_worker.py @@ -170,8 +170,8 @@ def start_worker( def spawn_worker( run_name: str, iteration: int, - raw_array_identifier: 'LocalArrayIdentifier', - prediction_array_identifier: 'LocalArrayIdentifier', + raw_array_identifier: "LocalArrayIdentifier", + prediction_array_identifier: "LocalArrayIdentifier", compute_context: ComputeContext = LocalTorch(), ): """Spawn a worker to predict on a given dataset. diff --git a/dacapo/blockwise/threshold_worker.py b/dacapo/blockwise/threshold_worker.py index acd2093f3..929eebbf0 100644 --- a/dacapo/blockwise/threshold_worker.py +++ b/dacapo/blockwise/threshold_worker.py @@ -73,8 +73,8 @@ def start_worker( def spawn_worker( - input_array_identifier: 'LocalArrayIdentifier', - output_array_identifier: 'LocalArrayIdentifier', + input_array_identifier: "LocalArrayIdentifier", + output_array_identifier: "LocalArrayIdentifier", threshold: float = 0.0, compute_context: ComputeContext = LocalTorch(), ): diff --git a/dacapo/experiments/tasks/post_processors/argmax_post_processor.py b/dacapo/experiments/tasks/post_processors/argmax_post_processor.py index d5b2361da..a3af2d62c 100644 --- a/dacapo/experiments/tasks/post_processors/argmax_post_processor.py +++ b/dacapo/experiments/tasks/post_processors/argmax_post_processor.py @@ -2,6 +2,7 @@ from dacapo.blockwise.scheduler import run_blockwise from dacapo.compute_context import ComputeContext, LocalTorch from dacapo.experiments.datasplits.datasets.arrays.zarr_array import ZarrArray +from dacapo.store.array_store import LocalArrayIdentifier from .argmax_post_processor_parameters import ArgmaxPostProcessorParameters from .post_processor import PostProcessor import numpy as np diff --git a/dacapo/experiments/tasks/post_processors/watershed_post_processor.py b/dacapo/experiments/tasks/post_processors/watershed_post_processor.py index bc1ce6c65..c447694ed 100644 --- a/dacapo/experiments/tasks/post_processors/watershed_post_processor.py +++ b/dacapo/experiments/tasks/post_processors/watershed_post_processor.py @@ -35,7 +35,7 @@ def set_prediction(self, prediction_array_identifier): def process( self, parameters: WatershedPostProcessorParameters, - output_array_identifier: 'LocalArrayIdentifier', + output_array_identifier: "LocalArrayIdentifier", ): output_array = ZarrArray.create_from_array_identifier( output_array_identifier, diff --git a/dacapo/predict.py b/dacapo/predict.py index 3e5dea79a..0e8dfaf45 100644 --- a/dacapo/predict.py +++ b/dacapo/predict.py @@ -5,8 +5,7 @@ from dacapo.experiments import Run from dacapo.gp import DaCapoArraySource from dacapo.experiments import Model -from dacapo.store import create_config_store -from dacapo.store import create_weights_store +from dacapo.store.create_store import create_config_store, create_weights_store from dacapo.store.local_array_store import LocalArrayIdentifier from dacapo.compute_context import LocalTorch, ComputeContext from dacapo.experiments.datasplits.datasets.arrays import ZarrArray, Array diff --git a/dacapo/store/__init__.py b/dacapo/store/__init__.py index 420359ae4..e69de29bb 100644 --- a/dacapo/store/__init__.py +++ b/dacapo/store/__init__.py @@ -1,7 +0,0 @@ -from .converter import converter -from .create_store import ( - create_array_store, - create_config_store, - create_stats_store, - create_weights_store, -) diff --git a/dacapo/store/array_store.py b/dacapo/store/array_store.py index 7fdc21adc..7c44ab7ab 100644 --- a/dacapo/store/array_store.py +++ b/dacapo/store/array_store.py @@ -62,7 +62,7 @@ def validation_input_arrays( pass @abstractmethod - def remove(self, array_identifier: 'LocalArrayIdentifier') -> None: + def remove(self, array_identifier: "LocalArrayIdentifier") -> None: """Remove an array by its identifier.""" pass diff --git a/dacapo/store/local_array_store.py b/dacapo/store/local_array_store.py index 21d3da0ab..73994d980 100644 --- a/dacapo/store/local_array_store.py +++ b/dacapo/store/local_array_store.py @@ -85,7 +85,7 @@ def validation_container(self, run_name: str) -> LocalContainerIdentifier: Path(self.__get_run_dir(run_name), "validation.zarr") ) - def remove(self, array_identifier: 'LocalArrayIdentifier') -> None: + def remove(self, array_identifier: "LocalArrayIdentifier") -> None: container = array_identifier.container dataset = array_identifier.dataset diff --git a/dacapo/train.py b/dacapo/train.py index 7beb096b4..5b387d339 100644 --- a/dacapo/train.py +++ b/dacapo/train.py @@ -1,7 +1,11 @@ -from dacapo.store.create_store import create_array_store -from .experiments import Run -from .compute_context import LocalTorch, ComputeContext -from .store import create_config_store, create_stats_store, create_weights_store +from dacapo.store.create_store import ( + create_array_store, + create_config_store, + create_stats_store, + create_weights_store, +) +from dacapo.experiments import Run +from dacapo.compute_context import LocalTorch, ComputeContext from .validate import validate_run import torch diff --git a/dacapo/validate.py b/dacapo/validate.py index a1cf9da7d..348549f32 100644 --- a/dacapo/validate.py +++ b/dacapo/validate.py @@ -2,7 +2,7 @@ from .compute_context import LocalTorch, ComputeContext from .experiments import Run, ValidationIterationScores from .experiments.datasplits.datasets.arrays import ZarrArray -from .store import ( +from .store.create_store import ( create_array_store, create_config_store, create_stats_store, diff --git a/tests/components/test_arrays.py b/tests/components/test_arrays.py index b81a4bc7e..d62dcb973 100644 --- a/tests/components/test_arrays.py +++ b/tests/components/test_arrays.py @@ -1,6 +1,6 @@ from ..fixtures import * -from dacapo.store import create_config_store +from dacapo.store.create_store import create_config_store import pytest from pytest_lazyfixture import lazy_fixture diff --git a/tests/components/test_trainers.py b/tests/components/test_trainers.py index 8ef792ba5..172a89b75 100644 --- a/tests/components/test_trainers.py +++ b/tests/components/test_trainers.py @@ -1,6 +1,6 @@ from ..fixtures import * -from dacapo.store import create_config_store +from dacapo.store.create_store import create_config_store import pytest from pytest_lazyfixture import lazy_fixture diff --git a/tests/operations/test_apply.py b/tests/operations/test_apply.py index 9a633a90a..53ca30b7f 100644 --- a/tests/operations/test_apply.py +++ b/tests/operations/test_apply.py @@ -2,7 +2,7 @@ from dacapo.experiments import Run from dacapo.compute_context import LocalTorch -from dacapo.store import create_config_store, create_weights_store +from dacapo.store.create_store import create_config_store, create_weights_store from dacapo import apply import pytest diff --git a/tests/operations/test_train.py b/tests/operations/test_train.py index abecfd9c3..846afe6c3 100644 --- a/tests/operations/test_train.py +++ b/tests/operations/test_train.py @@ -3,7 +3,7 @@ from dacapo.experiments import Run from dacapo.compute_context import LocalTorch -from dacapo.store import create_config_store, create_weights_store +from dacapo.store.create_store import create_config_store, create_weights_store from dacapo.train import train_run import pytest diff --git a/tests/operations/test_validate.py b/tests/operations/test_validate.py index d18a87197..54d6dc5e4 100644 --- a/tests/operations/test_validate.py +++ b/tests/operations/test_validate.py @@ -2,7 +2,7 @@ from dacapo.experiments import Run from dacapo.compute_context import LocalTorch -from dacapo.store import create_config_store, create_weights_store +from dacapo.store.create_store import create_config_store, create_weights_store from dacapo import validate import pytest From 04c5d94109ab3f6a2c884de963fa9038661b65a6 Mon Sep 17 00:00:00 2001 From: Diane Adjavon Date: Thu, 15 Feb 2024 11:40:51 -0500 Subject: [PATCH 08/15] chore: Clean up absolute vs relative imports --- .../tasks/post_processors/watershed_post_processor.py | 1 + dacapo/train.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dacapo/experiments/tasks/post_processors/watershed_post_processor.py b/dacapo/experiments/tasks/post_processors/watershed_post_processor.py index c447694ed..cb839630d 100644 --- a/dacapo/experiments/tasks/post_processors/watershed_post_processor.py +++ b/dacapo/experiments/tasks/post_processors/watershed_post_processor.py @@ -1,4 +1,5 @@ from dacapo.experiments.datasplits.datasets.arrays import ZarrArray +from dacapo.store.array_store import LocalArrayIdentifier from .watershed_post_processor_parameters import WatershedPostProcessorParameters from .post_processor import PostProcessor diff --git a/dacapo/train.py b/dacapo/train.py index 5b387d339..04642e4d3 100644 --- a/dacapo/train.py +++ b/dacapo/train.py @@ -6,7 +6,7 @@ ) from dacapo.experiments import Run from dacapo.compute_context import LocalTorch, ComputeContext -from .validate import validate_run +from dacapo.validate import validate_run import torch from tqdm import tqdm From 7d39cd51bd88603185127602c4392c78d82355e9 Mon Sep 17 00:00:00 2001 From: Diane Adjavon Date: Thu, 15 Feb 2024 11:44:41 -0500 Subject: [PATCH 09/15] fix: Add toml dependency --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1b9d716a2..22026a57a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,8 @@ dependencies = [ "xarray", "cattrs", "numpy-indexed", - "click",] + "click", + "toml"] # extras # https://peps.python.org/pep-0621/#dependencies-optional-dependencies From c96aa22a0d7f587f7f596c4bbd40dc49c6dc5ea2 Mon Sep 17 00:00:00 2001 From: Diane Adjavon Date: Thu, 15 Feb 2024 12:03:29 -0500 Subject: [PATCH 10/15] chore: Swap tests to pyproject's dev --- .github/workflows/tests.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 285b57402..dd06265ad 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -30,8 +30,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e . - pip install -r requirements-dev.txt + pip install -e ".[dev]" - name: Test run: pytest --color=yes --cov --cov-report=xml --cov-report=term-missing From 552f2eeb0f8e59a76f291c4d86537861114626c8 Mon Sep 17 00:00:00 2001 From: Diane Adjavon Date: Thu, 15 Feb 2024 12:06:15 -0500 Subject: [PATCH 11/15] chore: Add tests to on PRs and Push to dev/main --- .github/workflows/tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index dd06265ad..0c69a93c0 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -4,9 +4,11 @@ on: pull_request: branches: - main + - dev/main push: branches: - main + - dev/main workflow_dispatch: jobs: From 91a0f6343913882aab8e921074736fc9c3670e75 Mon Sep 17 00:00:00 2001 From: Jeff Rhoades <37990507+rhoadesScholar@users.noreply.github.com> Date: Thu, 15 Feb 2024 12:17:29 -0500 Subject: [PATCH 12/15] Update CONTRIBUTOR.md add dev/main to contrib docs --- CONTRIBUTOR.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTOR.md b/CONTRIBUTOR.md index 5e37cb8d5..5604ad317 100644 --- a/CONTRIBUTOR.md +++ b/CONTRIBUTOR.md @@ -23,5 +23,6 @@ This will also be run automatically when a PR is made to master and a codecov re ## Branching and PRs - Users that have been added to the CellMap organization and the DaCapo project should be able to develop directly into the CellMap fork of DaCapo. Other users will need to create a fork. -- For a completely new feature, make a branch off of the `main` branch of CellMap's fork of DaCapo with a name describing the feature. If you are collaborating on a feature that already has a branch, you can branch off that feature branch. -- Currently, you should make your PRs into the main branch of CellMap's fork, or the feature branch you branched off of. PRs currently require one maintainer's approval before merging. Once the PR is merged, the feature branch should be deleted. +- For a completely new feature, make a branch off of the `dev/main` branch of CellMap's fork of DaCapo with a name describing the feature. If you are collaborating on a feature that already has a branch, you can branch off that feature branch. +- Currently, you should make your PRs into the `dev/main` branch of CellMap's fork, or the feature branch you branched off of. PRs currently require one maintainer's approval before merging. Once the PR is merged, the feature branch should be deleted. +- `dev/main` will be regularly merged to `main` when new features are fully implemented and all tests are passing. From 30ffdbb668d852e2e4057867d10b684e36df14ab Mon Sep 17 00:00:00 2001 From: Diane Adjavon Date: Thu, 15 Feb 2024 12:32:54 -0500 Subject: [PATCH 13/15] fix: Add pytest to developper requirements --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 80b70dcda..33169637b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ dev = [ "ruff", "pre-commit", "jupyter", + "pytest" ] docs = [ "sphinx-autodoc-typehints", From 3d762d0fc75444ab60f753feb3400f1c6054ff1a Mon Sep 17 00:00:00 2001 From: Diane Adjavon Date: Thu, 15 Feb 2024 12:51:24 -0500 Subject: [PATCH 14/15] fix: Use test install for tests instead of dev --- .github/workflows/tests.yaml | 2 +- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 569147a9b..0da8db384 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -32,7 +32,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ".[dev]" + pip install -e ".[test]" - name: Test run: pytest --color=yes --cov --cov-report=xml --cov-report=term-missing diff --git a/pyproject.toml b/pyproject.toml index 60b8c9954..18af7fdfa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,6 @@ dev = [ "ruff", "pre-commit", "jupyter", - "pytest" ] docs = [ "sphinx-autodoc-typehints", From 71f47d5e4a4db7846f300e9705280bbd6a6a194e Mon Sep 17 00:00:00 2001 From: Diane Adjavon Date: Thu, 15 Feb 2024 12:59:15 -0500 Subject: [PATCH 15/15] chore: Add context around file opening for autoclose --- dacapo/store/file_config_store.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dacapo/store/file_config_store.py b/dacapo/store/file_config_store.py index 5fbe1ca5c..09f8215cd 100644 --- a/dacapo/store/file_config_store.py +++ b/dacapo/store/file_config_store.py @@ -98,10 +98,12 @@ def __save_insert(self, collection, data, ignore=None): file_store = collection / f"{name}.toml" if not file_store.exists(): - toml.dump(dict(data), file_store.open("w")) + with file_store.open("w") as f: + toml.dump(dict(data), f) else: - existing = toml.load(file_store.open("r")) + with file_store.open("r") as f: + existing = toml.load(f) if not self.__same_doc(existing, data, ignore): raise DuplicateNameError( @@ -113,7 +115,8 @@ def __save_insert(self, collection, data, ignore=None): def __load(self, collection, name): file_store = collection / f"{name}.toml" if file_store.exists(): - return toml.load(file_store.open("r")) + with file_store.open("r") as f: + return toml.load(f) else: raise ValueError(f"No config with name: {name} in collection: {collection}")