diff --git a/dacapo/store/create_store.py b/dacapo/store/create_store.py index 57eed6d9b..e04060e90 100644 --- a/dacapo/store/create_store.py +++ b/dacapo/store/create_store.py @@ -31,7 +31,7 @@ def create_config_store(): db_name = options.mongo_db_name return MongoConfigStore(db_host, db_name) elif options.type == "files": - store_path = Path(options.runs_base_dir).expanduser() + store_path = Path(options.runs_base_dir) return FileConfigStore(store_path / "configs") else: raise ValueError(f"Unknown store type {options.type}") @@ -62,7 +62,7 @@ def create_stats_store(): db_name = options.mongo_db_name return MongoStatsStore(db_host, db_name) elif options.type == "files": - store_path = Path(options.runs_base_dir).expanduser() + store_path = Path(options.runs_base_dir) return FileStatsStore(store_path / "stats") else: raise ValueError(f"Unknown store type {options.type}") @@ -85,8 +85,7 @@ def create_weights_store(): options = Options.instance() - # currently, only the LocalWeightsStore is supported - base_dir = Path(options.runs_base_dir).expanduser() + base_dir = Path(options.runs_base_dir) return LocalWeightsStore(base_dir) diff --git a/examples/aws/README.md b/examples/aws/README.md new file mode 100644 index 000000000..96f8c9499 --- /dev/null +++ b/examples/aws/README.md @@ -0,0 +1,14 @@ +You can work locally using S3 data by setting the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables. You can also set the `AWS_REGION` environment variable to specify the region to use. If you are using a profile, you can set the `AWS_PROFILE` environment variable to specify the profile to use. + +```bash +aws configure +``` + +In order to store checkpoints and experiments data in S3, you need to modify `dacapo.yaml` to include the following: + +```yaml +runs_base_dir: "s3://dacapotest" +``` + +For configs and stats, you can save them locally or s3 by setting `type: files` or for mongodb by setting `type: mongo` in the `dacapo.yaml` file. + diff --git a/examples/aws/aws_store_check.py b/examples/aws/aws_store_check.py new file mode 100644 index 000000000..f44b261ed --- /dev/null +++ b/examples/aws/aws_store_check.py @@ -0,0 +1,30 @@ +# %% +import dacapo + +# from import create_config_store + +config_store = dacapo.store.create_store.create_config_store() + +# %% +from dacapo import Options + +options = Options.instance() + +# %% +options +# %% +from dacapo.experiments.tasks import DistanceTaskConfig + +task_config = DistanceTaskConfig( + name="cosem_distance_task_4nm", + channels=["mito"], + clip_distance=40.0, + tol_distance=40.0, + scale_factor=80.0, +) + +# %% + +config_store.store_task_config(task_config) + +# %% diff --git a/examples/aws/cloud_csv.csv b/examples/aws/cloud_csv.csv new file mode 100644 index 000000000..99a407a82 --- /dev/null +++ b/examples/aws/cloud_csv.csv @@ -0,0 +1,3 @@ +train,s3://janelia-cosem-datasets/jrc_hela-2/jrc_hela-2.zarr,recon-1/em/fibsem-uint8,s3://janelia-cosem-datasets/jrc_hela-2/jrc_hela-2.zarr,recon-1/labels/groundtruth/crop155/[nuc] +train,s3://janelia-cosem-datasets/jrc_hela-2/jrc_hela-2.zarr,recon-1/em/fibsem-uint8,s3://janelia-cosem-datasets/jrc_hela-2/jrc_hela-2.zarr,recon-1/labels/groundtruth/crop7/[nuc] +val,s3://janelia-cosem-datasets/jrc_hela-2/jrc_hela-2.zarr,recon-1/em/fibsem-uint8,s3://janelia-cosem-datasets/jrc_hela-2/jrc_hela-2.zarr,recon-1/labels/groundtruth/crop6/[nuc] \ No newline at end of file diff --git a/examples/aws/dacapo.yaml b/examples/aws/dacapo.yaml new file mode 100644 index 000000000..960719a6d --- /dev/null +++ b/examples/aws/dacapo.yaml @@ -0,0 +1,3 @@ + +runs_base_dir: "s3://dacapotest" +type: "files" diff --git a/examples/aws/s3_datasplit.py b/examples/aws/s3_datasplit.py new file mode 100644 index 000000000..f5bb72b79 --- /dev/null +++ b/examples/aws/s3_datasplit.py @@ -0,0 +1,16 @@ +# %% +from dacapo.experiments.datasplits import DataSplitGenerator +from funlib.geometry import Coordinate + +input_resolution = Coordinate(8, 8, 8) +output_resolution = Coordinate(4, 4, 4) +datasplit_config = DataSplitGenerator.generate_from_csv( + "cloud_csv.csv", + input_resolution, + output_resolution, +).compute() +# %% +datasplit = datasplit_config.datasplit_type(datasplit_config) +# %% +viewer = datasplit._neuroglancer() +# %% diff --git a/pyproject.toml b/pyproject.toml index 0ef3a39c2..0ab64cdff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,8 @@ dependencies = [ "click", "pyyaml", "scipy", + "upath", + "boto3", ] # extras