From f753c3dfc61d841bbc27702b84d134a0587a56fb Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Wed, 15 May 2024 22:17:09 -0400 Subject: [PATCH] Separate sanity checks in N5Client more carefully --- .../render/client/spark/n5/N5Client.java | 69 ++++++++++--------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java index d3a1d0bde..fd5944d78 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java @@ -332,6 +332,8 @@ public void run() } else if (parameters.appendToExisting) { + checkConsistency(fullScaleDatasetName, dimensions, blockSize); + if (fullScaleDatasetName.endsWith("s0")) { final File parentDir = datasetDir.getParentFile(); final File[] downsampledDirs = parentDir.listFiles(DOWNSAMPLED_DIR_FILTER); @@ -410,6 +412,35 @@ public void run() sparkContext.close(); } + private void checkConsistency(final String fullScaleDatasetName, final long[] dimensions, final int[] blockSize) { + try (final N5Writer n5 = new N5FSWriter(parameters.n5Path)) { + final DatasetAttributes datasetAttributes = n5.getDatasetAttributes(fullScaleDatasetName); + final DataType dataType = getDataType(); + + if (! Arrays.equals(blockSize, datasetAttributes.getBlockSize())) { + throw new IllegalArgumentException("append blockSize " + Arrays.toString(blockSize) + " does not match existing dataset block size " + Arrays.toString(datasetAttributes.getBlockSize())); + } + if (dataType != datasetAttributes.getDataType()) { + throw new IllegalArgumentException("append dataType " + dataType + " does not match existing dataset data type " + datasetAttributes.getDataType()); + } + final long[] existingDimensions = datasetAttributes.getDimensions(); + if (dimensions.length != existingDimensions.length) { + throw new IllegalArgumentException("append dimensions " + Arrays.toString(dimensions) + " differ in length from existing dataset dimensions " + Arrays.toString(existingDimensions)); + } + if (dimensions.length != 3) { + throw new IllegalArgumentException("append export not supported for " + dimensions.length + "D volumes"); + } + for (int i = 0; i < dimensions.length; i++) { + if (dimensions[i] < existingDimensions[i]) { + throw new IllegalArgumentException("append dimension " + i + " has shrunk, append dimensions are " + Arrays.toString(dimensions) + " but existing dataset dimensions are " + Arrays.toString(existingDimensions)); + } + } + if (dimensions[2] == existingDimensions[2]) { + throw new IllegalArgumentException("nothing new to export since last z remains " + dimensions[2]); + } + } + } + public void renderStack(final JavaSparkContext sparkContext, final int[] blockSize, final String fullScaleDatasetName, @@ -494,43 +525,15 @@ public static long setupAppendExportN5(final Parameters parameters, final long minZToRender; try (final N5Writer n5 = new N5FSWriter(parameters.n5Path)) { + final DatasetAttributes existingAttributes = n5.getDatasetAttributes(fullScaleDatasetName); + final DatasetAttributes newDatasetAttributes = new DatasetAttributes(dimensions, blockSize, dataType, existingAttributes.getCompression()); - final DatasetAttributes datasetAttributes = n5.getDatasetAttributes(fullScaleDatasetName); - - if (! Arrays.equals(blockSize, datasetAttributes.getBlockSize())) { - throw new IllegalArgumentException("append blockSize " + Arrays.toString(blockSize) + " does not match existing dataset block size " + Arrays.toString(datasetAttributes.getBlockSize())); - } - if (dataType != datasetAttributes.getDataType()) { - throw new IllegalArgumentException("append dataType " + dataType + " does not match existing dataset data type " + datasetAttributes.getDataType()); - } - final long[] existingDimensions = datasetAttributes.getDimensions(); - if (dimensions.length != existingDimensions.length) { - throw new IllegalArgumentException("append dimensions " + Arrays.toString(dimensions) + " differ in length from existing dataset dimensions " + Arrays.toString(existingDimensions)); - } - if (dimensions.length != 3) { - throw new IllegalArgumentException("append export not supported for " + dimensions.length + "D volumes"); - } - for (int i = 0; i < dimensions.length; i++) { - if (dimensions[i] < existingDimensions[i]) { - throw new IllegalArgumentException("append dimension " + i + " has shrunk, append dimensions are " + Arrays.toString(dimensions) + " but existing dataset dimensions are " + Arrays.toString(existingDimensions)); - } - } - if (dimensions[2] == existingDimensions[2]) { - throw new IllegalArgumentException("nothing new to export since last z remains " + dimensions[2]); - } - - n5.setDatasetAttributes(fullScaleDatasetName, - new DatasetAttributes(dimensions, - blockSize, - dataType, - datasetAttributes.getCompression())); + n5.setDatasetAttributes(fullScaleDatasetName, newDatasetAttributes); + final long[] existingDimensions = existingAttributes.getDimensions(); minZToRender = existingDimensions[2] + 1; } - updateFullScaleExportAttributes(parameters, - fullScaleDatasetName, - stackMetaData); - + updateFullScaleExportAttributes(parameters, fullScaleDatasetName, stackMetaData); LOG.info("setupAppendExportN5: returning minZToRender value of {}, maxZ is {}", minZToRender, dimensions[2]); return minZToRender;