fixes layer detections (#163)

* fixes layer detections
* PR feedback
* try-except
* Merge branch 'master' into fix-metadata

requirements.txt

@@ -5,7 +5,7 @@ pyyaml
 pytest
 wkw==0.1.1
 requests
-black
+black==19.3b0
 cluster_tools==1.44
 natsort
 psutil

wkcuber/cubing.py

@@ -45,7 +45,7 @@ def create_parser():
     )
 
     parser.add_argument(
-        "--start_z", help="The z coordinate of the first slice", default=0, type=int,
+        "--start_z", help="The z coordinate of the first slice", default=0, type=int
     )
 
     parser.add_argument(

wkcuber/mag.py

@@ -47,7 +47,11 @@ def to_layer_name(self):
         if x == y and y == z:
             return str(x)
         else:
-            return "{}-{}-{}".format(x, y, z)
+            return self.to_long_layer_name()
+
+    def to_long_layer_name(self):
+        x, y, z = self.mag
+        return "{}-{}-{}".format(x, y, z)
 
     def to_array(self):
         return self.mag

wkcuber/metadata.py

@@ -178,9 +178,19 @@ def convert_dtype_to_element_class(dtype):
     return conversion_map.get(dtype, str(dtype))
 
 
-def detect_dtype(dataset_path, layer, mag: Mag = Mag(1)):
+def detect_mag_path(dataset_path, layer, mag: Mag = Mag(1)):
     layer_path = path.join(dataset_path, layer, str(mag))
     if path.exists(layer_path):
+        return layer_path
+    layer_path = path.join(dataset_path, layer, mag.to_long_layer_name())
+    if path.exists(layer_path):
+        return layer_path
+    return None
+
+
+def detect_dtype(dataset_path, layer, mag: Mag = Mag(1)):
+    layer_path = detect_mag_path(dataset_path, layer, mag)
+    if layer_path is not None:
         with wkw.Dataset.open(layer_path) as dataset:
             voxel_size = dataset.header.voxel_type
             num_channels = dataset.header.num_channels
@@ -191,16 +201,18 @@ def detect_dtype(dataset_path, layer, mag: Mag = Mag(1)):
 
 
 def detect_cubeLength(dataset_path, layer, mag: Mag = Mag(1)):
-    layer_path = path.join(dataset_path, layer, str(mag))
-    if path.exists(layer_path):
+    layer_path = detect_mag_path(dataset_path, layer, mag)
+    if layer_path is not None:
         with wkw.Dataset.open(layer_path) as dataset:
             return dataset.header.block_len * dataset.header.file_len
 
 
 def detect_bbox(dataset_path, layer, mag: Mag = Mag(1)):
     # Detect the coarse bounding box of a dataset by iterating
     # over the WKW cubes
-    layer_path = path.join(dataset_path, layer, str(mag))
+    layer_path = detect_mag_path(dataset_path, layer, mag)
+    if layer_path is None:
+        return None
 
     def list_files(layer_path):
         return iglob(path.join(layer_path, "*", "*", "*.wkw"), recursive=True)
@@ -219,6 +231,8 @@ def list_cubes(layer_path):
     max_x, max_y, max_z = max(xs), max(ys), max(zs)
 
     cubeLength = detect_cubeLength(dataset_path, layer, mag)
+    if cubeLength is None:
+        return None
 
     return {
         "topLeft": [min_x * cubeLength, min_y * cubeLength, min_z * cubeLength],
@@ -243,12 +257,15 @@ def detect_standard_layer(
 
     mags = list(detect_resolutions(dataset_path, layer_name))
     mags = sorted(mags)
-    assert len(mags) > 0, "No resolutions found"
+    assert len(mags) > 0, f"No resolutions found for {dataset_path}/{layer_name}"
 
     if exact_bounding_box is None:
         bbox = detect_bbox(dataset_path, layer_name, mags[0])
     else:
         bbox = exact_bounding_box
+    assert (
+        bbox is not None
+    ), f"Could not detect bounding box for {dataset_path}/{layer_name}"
 
     # BB can be created manually
     # assert the presence/spelling of all attributes
@@ -266,8 +283,13 @@ def detect_standard_layer(
         }
         for mag in mags
     ]
+    resolutions = [r for r in resolutions if r["cubeLength"] is not None]
+    assert len(resolutions) > 0, f"No resolutions found for {dataset_path}/{layer_name}"
 
     dtype = detect_dtype(dataset_path, layer_name, mags[0])
+    assert (
+        dtype is not None
+    ), f"Data type could not be detected for {dataset_path}/{layer_name}"
 
     return {
         "dataFormat": "wkw",
@@ -299,7 +321,7 @@ def detect_segmentation_layer(
         logging.info("Computing max id of layer={}".format(layer_name))
         # Computing the current largest segment id
         # This may take very long due to IO load
-        layer_path = path.join(dataset_path, layer_name, "1")
+        layer_path = detect_mag_path(dataset_path, layer_name, Mag(1))
         with wkw.Dataset.open(layer_path) as dataset:
             bbox = layer_info["boundingBox"]
             layer_info["largestSegmentId"] = int(
@@ -335,7 +357,16 @@ def detect_layers(dataset_path: str, max_id, compute_max_id, exact_bounding_box=
     for layer_name in available_layer_names:
         # color and segmentation are already checked explicitly to ensure downwards compatibility (some older datasets don't have the header.wkw file)
         if layer_name not in ["color", "segmentation"]:
-            yield detect_standard_layer(dataset_path, layer_name, exact_bounding_box)
+            try:
+                layer_info = detect_standard_layer(
+                    dataset_path, layer_name, exact_bounding_box
+                )
+            except:
+                pass
+            if layer_info is not None:
+                yield layer_info
+            else:
+                logging.warning(f"{layer_name} is not a WKW layer")
 
 
 if __name__ == "__main__":