Merge branch 's3' of github.com:janelia-cellmap/dacapo into s3

dacapo/options.py

@@ -35,21 +35,6 @@ class DaCapoConfig:
             "Currently, only 'files' and 'mongo' are supported with files being the default."
         },
     )
-
-    store: Optional[str] = attr.ib(
-        default="local",
-        metadata={
-            "help_text": "The type of store to use for storing configurations and statistics. "
-            "Currently, only 'local' and 's3' are supported with local being the default."
-        },
-    )
-    s3_bucket: Optional[str] = attr.ib(
-        default=None,
-        metadata={
-            "help_text": "The S3 bucket to use for storing configurations and statistics."
-        },
-    )
-
     runs_base_dir: Path = attr.ib(
         default=Path(expanduser("~/dacapo")),
         metadata={

dacapo/store/create_store.py

@@ -1,6 +1,5 @@
 from .local_array_store import LocalArrayStore
 from .local_weights_store import LocalWeightsStore
-from .s3_weights_store import S3WeightsStore
 from .mongo_config_store import MongoConfigStore
 from .file_config_store import FileConfigStore
 from .mongo_stats_store import MongoStatsStore
@@ -32,7 +31,7 @@ def create_config_store():
         db_name = options.mongo_db_name
         return MongoConfigStore(db_host, db_name)
     elif options.type == "files":
-        store_path = Path(options.runs_base_dir).expanduser()
+        store_path = Path(options.runs_base_dir)
         return FileConfigStore(store_path / "configs")
     else:
         raise ValueError(f"Unknown store type {options.type}")
@@ -63,7 +62,7 @@ def create_stats_store():
         db_name = options.mongo_db_name
         return MongoStatsStore(db_host, db_name)
     elif options.type == "files":
-        store_path = Path(options.runs_base_dir).expanduser()
+        store_path = Path(options.runs_base_dir)
         return FileStatsStore(store_path / "stats")
     else:
         raise ValueError(f"Unknown store type {options.type}")
@@ -86,14 +85,8 @@ def create_weights_store():
 
     options = Options.instance()
 
-    if options.store == "s3":
-        s3_bucket = options.s3_bucket
-        return S3WeightsStore(s3_bucket)
-    elif options.store == "local":
-        base_dir = Path(options.runs_base_dir).expanduser()
-        return LocalWeightsStore(base_dir)
-    else:
-        raise ValueError(f"Unknown weights store type {options.type}")
+    base_dir = Path(options.runs_base_dir)
+    return LocalWeightsStore(base_dir)
 
 
 def create_array_store():

examples/aws/README.md

@@ -0,0 +1,14 @@
+You can work locally using S3 data by setting the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables. You can also set the `AWS_REGION` environment variable to specify the region to use. If you are using a profile, you can set the `AWS_PROFILE` environment variable to specify the profile to use.
+
+```bash
+aws configure
+```
+
+In order to store checkpoints and experiments data in S3, you need to modify `dacapo.yaml` to include the following:
+
+```yaml
+runs_base_dir: "s3://dacapotest"
+```
+
+For configs and stats, you can save them locally or s3 by setting `type: files` or for mongodb by setting `type: mongo` in the `dacapo.yaml` file.
+

examples/aws/aws_store_check.py

@@ -0,0 +1,30 @@
+# %%
+import dacapo
+
+# from  import create_config_store
+
+config_store = dacapo.store.create_store.create_config_store()
+
+# %%
+from dacapo import Options
+
+options = Options.instance()
+
+# %%
+options
+# %%
+from dacapo.experiments.tasks import DistanceTaskConfig
+
+task_config = DistanceTaskConfig(
+    name="cosem_distance_task_4nm",
+    channels=["mito"],
+    clip_distance=40.0,
+    tol_distance=40.0,
+    scale_factor=80.0,
+)
+
+# %%
+
+config_store.store_task_config(task_config)
+
+# %%

examples/aws/dacapo.yaml

@@ -1,6 +1,3 @@
 
-store: "s3"
-s3_bucket: "s3//dacapo-tests"
-
+runs_base_dir: "s3://dacapotest"
 type: "files"
-

examples/aws/s3_datasplit.py

@@ -10,7 +10,7 @@
     output_resolution,
 ).compute()
 # %%
-datasplit = datasplit_config.datasplit_type(data)
+datasplit = datasplit_config.datasplit_type(datasplit_config)
 # %%
 viewer = datasplit._neuroglancer()
 # %%