From 6135d88c8d8c43b2298ad2fbd5a05e149400c447 Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Thu, 28 Nov 2019 14:05:37 +0100 Subject: [PATCH] fixes layer detections (#163) * fixes layer detections * PR feedback * try-except * Merge branch 'master' into fix-metadata --- requirements.txt | 2 +- wkcuber/cubing.py | 2 +- wkcuber/mag.py | 6 +++++- wkcuber/metadata.py | 45 ++++++++++++++++++++++++++++++++++++++------- 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5f2d8443b..d8655d4a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ pyyaml pytest wkw==0.1.1 requests -black +black==19.3b0 cluster_tools==1.44 natsort psutil diff --git a/wkcuber/cubing.py b/wkcuber/cubing.py index dc9d603bc..25b35c7d7 100644 --- a/wkcuber/cubing.py +++ b/wkcuber/cubing.py @@ -45,7 +45,7 @@ def create_parser(): ) parser.add_argument( - "--start_z", help="The z coordinate of the first slice", default=0, type=int, + "--start_z", help="The z coordinate of the first slice", default=0, type=int ) parser.add_argument( diff --git a/wkcuber/mag.py b/wkcuber/mag.py index ea494abdf..fae950071 100644 --- a/wkcuber/mag.py +++ b/wkcuber/mag.py @@ -47,7 +47,11 @@ def to_layer_name(self): if x == y and y == z: return str(x) else: - return "{}-{}-{}".format(x, y, z) + return self.to_long_layer_name() + + def to_long_layer_name(self): + x, y, z = self.mag + return "{}-{}-{}".format(x, y, z) def to_array(self): return self.mag diff --git a/wkcuber/metadata.py b/wkcuber/metadata.py index a8fd65f27..d492d5c83 100644 --- a/wkcuber/metadata.py +++ b/wkcuber/metadata.py @@ -178,9 +178,19 @@ def convert_dtype_to_element_class(dtype): return conversion_map.get(dtype, str(dtype)) -def detect_dtype(dataset_path, layer, mag: Mag = Mag(1)): +def detect_mag_path(dataset_path, layer, mag: Mag = Mag(1)): layer_path = path.join(dataset_path, layer, str(mag)) if path.exists(layer_path): + return layer_path + layer_path = path.join(dataset_path, layer, mag.to_long_layer_name()) + if path.exists(layer_path): + return layer_path + return None + + +def detect_dtype(dataset_path, layer, mag: Mag = Mag(1)): + layer_path = detect_mag_path(dataset_path, layer, mag) + if layer_path is not None: with wkw.Dataset.open(layer_path) as dataset: voxel_size = dataset.header.voxel_type num_channels = dataset.header.num_channels @@ -191,8 +201,8 @@ def detect_dtype(dataset_path, layer, mag: Mag = Mag(1)): def detect_cubeLength(dataset_path, layer, mag: Mag = Mag(1)): - layer_path = path.join(dataset_path, layer, str(mag)) - if path.exists(layer_path): + layer_path = detect_mag_path(dataset_path, layer, mag) + if layer_path is not None: with wkw.Dataset.open(layer_path) as dataset: return dataset.header.block_len * dataset.header.file_len @@ -200,7 +210,9 @@ def detect_cubeLength(dataset_path, layer, mag: Mag = Mag(1)): def detect_bbox(dataset_path, layer, mag: Mag = Mag(1)): # Detect the coarse bounding box of a dataset by iterating # over the WKW cubes - layer_path = path.join(dataset_path, layer, str(mag)) + layer_path = detect_mag_path(dataset_path, layer, mag) + if layer_path is None: + return None def list_files(layer_path): return iglob(path.join(layer_path, "*", "*", "*.wkw"), recursive=True) @@ -219,6 +231,8 @@ def list_cubes(layer_path): max_x, max_y, max_z = max(xs), max(ys), max(zs) cubeLength = detect_cubeLength(dataset_path, layer, mag) + if cubeLength is None: + return None return { "topLeft": [min_x * cubeLength, min_y * cubeLength, min_z * cubeLength], @@ -243,12 +257,15 @@ def detect_standard_layer( mags = list(detect_resolutions(dataset_path, layer_name)) mags = sorted(mags) - assert len(mags) > 0, "No resolutions found" + assert len(mags) > 0, f"No resolutions found for {dataset_path}/{layer_name}" if exact_bounding_box is None: bbox = detect_bbox(dataset_path, layer_name, mags[0]) else: bbox = exact_bounding_box + assert ( + bbox is not None + ), f"Could not detect bounding box for {dataset_path}/{layer_name}" # BB can be created manually # assert the presence/spelling of all attributes @@ -266,8 +283,13 @@ def detect_standard_layer( } for mag in mags ] + resolutions = [r for r in resolutions if r["cubeLength"] is not None] + assert len(resolutions) > 0, f"No resolutions found for {dataset_path}/{layer_name}" dtype = detect_dtype(dataset_path, layer_name, mags[0]) + assert ( + dtype is not None + ), f"Data type could not be detected for {dataset_path}/{layer_name}" return { "dataFormat": "wkw", @@ -299,7 +321,7 @@ def detect_segmentation_layer( logging.info("Computing max id of layer={}".format(layer_name)) # Computing the current largest segment id # This may take very long due to IO load - layer_path = path.join(dataset_path, layer_name, "1") + layer_path = detect_mag_path(dataset_path, layer_name, Mag(1)) with wkw.Dataset.open(layer_path) as dataset: bbox = layer_info["boundingBox"] layer_info["largestSegmentId"] = int( @@ -335,7 +357,16 @@ def detect_layers(dataset_path: str, max_id, compute_max_id, exact_bounding_box= for layer_name in available_layer_names: # color and segmentation are already checked explicitly to ensure downwards compatibility (some older datasets don't have the header.wkw file) if layer_name not in ["color", "segmentation"]: - yield detect_standard_layer(dataset_path, layer_name, exact_bounding_box) + try: + layer_info = detect_standard_layer( + dataset_path, layer_name, exact_bounding_box + ) + except: + pass + if layer_info is not None: + yield layer_info + else: + logging.warning(f"{layer_name} is not a WKW layer") if __name__ == "__main__":