diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/NeuroglancerAttributes.java b/render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java similarity index 99% rename from render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/NeuroglancerAttributes.java rename to render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java index 4ddf98f4b..c389eb275 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/NeuroglancerAttributes.java +++ b/render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java @@ -1,4 +1,4 @@ -package org.janelia.render.client.spark.n5; +package org.janelia.alignment.util; import java.io.IOException; import java.nio.file.Path; diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/CrossCorrelationWithNextRegionalDataN5Writer.java b/render-ws-java-client/src/main/java/org/janelia/render/client/CrossCorrelationWithNextRegionalDataN5Writer.java new file mode 100644 index 000000000..9a2a8dee8 --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/CrossCorrelationWithNextRegionalDataN5Writer.java @@ -0,0 +1,183 @@ +package org.janelia.render.client; + +import java.io.IOException; +import java.io.Serializable; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.DoubleSummaryStatistics; +import java.util.List; + +import org.janelia.alignment.spec.Bounds; +import org.janelia.alignment.spec.stack.StackMetaData; +import org.janelia.alignment.util.NeuroglancerAttributes; +import org.janelia.render.client.zspacing.CrossCorrelationWithNextRegionalData; +import org.janelia.saalfeldlab.n5.DataType; +import org.janelia.saalfeldlab.n5.GzipCompression; +import org.janelia.saalfeldlab.n5.N5FSWriter; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.imglib2.N5Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import net.imglib2.Cursor; +import net.imglib2.img.array.ArrayImg; +import net.imglib2.img.array.ArrayImgs; +import net.imglib2.img.basictypeaccess.array.ByteArray; +import net.imglib2.type.numeric.integer.UnsignedByteType; + +public class CrossCorrelationWithNextRegionalDataN5Writer + implements Serializable { + + public static void writeN5(final List dataList, + final String basePath, + final String datasetName, + final StackMetaData stackMetaData) { + + final int numberOfLayers = dataList.size(); + LOG.info("writeN5: entry, processing {} z layers", numberOfLayers); + + final CrossCorrelationWithNextRegionalData firstLayerData = dataList.get(0); + final int rowCount = firstLayerData.getRegionalRowCount(); + final int columnCount = firstLayerData.getRegionalColumnCount(); + final Bounds stackBounds = stackMetaData.getStats().getStackBounds(); + + if ((rowCount > 0) && (columnCount >0)) { + + // need to write z layers concurrently, so make z block size 1 + final int[] blockSize = { columnCount, rowCount, 1 }; + final long[] blockDimensions = { blockSize[0], blockSize[1], blockSize[2] }; + final long[] datasetDimensions = { blockSize[0], blockSize[1], (long) stackBounds.getDeltaZ() }; + + try (final N5Writer n5Writer = new N5FSWriter(basePath)) { + + n5Writer.createDataset(datasetName, + datasetDimensions, + blockSize, + DataType.UINT8, + new GzipCompression()); + + final int margin = 55; + final int minViz = 0; + final int maxViz = 255 - margin; + final int vizRange = maxViz - minViz; + + for (int i = 0; i < numberOfLayers; i++) { + final CrossCorrelationWithNextRegionalData ccWithNext = dataList.get(i); + final long[] gridOffset = { 0, 0, (long) (ccWithNext.getpZ() - stackBounds.getMinZ()) }; + final ArrayImg block = ArrayImgs.unsignedBytes(blockDimensions); + final Cursor out = block.cursor(); + + final double[][] regionalCorrelation = ccWithNext.getRegionalCorrelation(); + + final DoubleSummaryStatistics nonZeroCorrelationStats = + Arrays.stream(regionalCorrelation) + .flatMapToDouble(Arrays::stream) + .filter(c -> c > 0.0) + .summaryStatistics(); + + final double minCc = nonZeroCorrelationStats.getMin(); + final double ccRange = nonZeroCorrelationStats.getMax() - minCc; + + // System.out.println("stats=" + nonZeroCorrelationStats + ", ccRange=" + ccRange + ", vizRange=" + vizRange); + + for (int row = 0; row < rowCount; row++) { + for (int column = 0; column < columnCount; column++) { + final UnsignedByteType outValue = out.next(); + final double correlation = regionalCorrelation[row][column]; + if (correlation > 0.0) { + final double relativeCc = correlation - minCc; + final double relativeViz = relativeCc * vizRange / ccRange; + //final double invertedRelativeViz = vizRange - relativeViz; // make worse correlation brighter + final int viz = (int) relativeViz + margin; + // System.out.println("[" + outValue.getIndex() + "] => " + correlation + ", " + viz); + outValue.set(viz); + } + } + } + + N5Utils.saveNonEmptyBlock(block, n5Writer, datasetName, gridOffset, new UnsignedByteType()); + + if (i % 5 == 0) { + LOG.info("writeN5: saved blocks for {} out of {} z layers", (i+1), numberOfLayers); + } + } + + final List stackResolutionValues = stackMetaData.getCurrentResolutionValues(); + final double pixelsPerColumn = stackBounds.getWidth() / (double) columnCount; + final double columnResolution = pixelsPerColumn * stackResolutionValues.get(0); + final double pixelsPerRow = stackBounds.getHeight() / (double) rowCount; + final double rowResolution = pixelsPerRow * stackResolutionValues.get(1); + final List ccVolumeResolutionValues = + Arrays.asList(columnResolution, rowResolution, stackResolutionValues.get(2)); + + // TODO: figure out why neuroglancer ignores this for cc data n5 + final List translationList = Arrays.asList(stackBounds.getMinX().longValue(), + stackBounds.getMinY().longValue(), + stackBounds.getMinZ().longValue()); + + // save additional parameters so that n5 can be viewed in neuroglancer + final NeuroglancerAttributes ngAttributes = + new NeuroglancerAttributes(ccVolumeResolutionValues, + "nm", + 0, + new int[0], + translationList, + NeuroglancerAttributes.NumpyContiguousOrdering.FORTRAN); + + ngAttributes.write(Paths.get(basePath), + Paths.get(datasetName)); + + } catch (final IOException e) { + throw new RuntimeException(e); + } + + } + + LOG.info("writeN5: exit"); + } + + @SuppressWarnings("CommentedOutCode") + public static void main(final String[] args) { +/* + try { +// final String baseOutputPath = "/Users/trautmane/Desktop/test-ic.n5"; +// final String zCorrDir = "/Users/trautmane/Desktop/zcorr"; +// final String owner = "hess_wafer_53"; +// final String project = "cut_000_to_009"; +// final String stack = "c000_s095_v01_align"; +// final String run = "run_20230726_161543"; + + + final String baseOutputPath = "/nrs/cellmap/data/jrc_ut23-0590-001/jrc_ut23-0590-001.n5"; + final String zCorrDir = "/nrs/cellmap/data/jrc_ut23-0590-001/z_corr"; + final String owner = "cellmap"; + final String project = "jrc_ut23_0590_001"; + final String stack = "v1_acquire_align"; + final String run = "run_20230803_092548_69_z_corr"; + + final Path runPath = Paths.get(zCorrDir, owner, project, stack, run); + final List dataList = + CrossCorrelationWithNextRegionalData.loadDataDirectory(runPath); + + final RenderDataClient renderDataClient = + new RenderDataClient("http://renderer-dev.int.janelia.org:8080/render-ws/v1", + owner, + project); + final StackMetaData stackMetaData = renderDataClient.getStackMetaData(stack); + final String datasetName = "/cc/regional/" + project + "/" + stack + "/" + run + "/s0"; + + writeN5(dataList, + baseOutputPath, + datasetName, + stackMetaData); + } catch (final Throwable t) { + t.printStackTrace(); + } + +*/ + } + + private static final Logger LOG = LoggerFactory.getLogger(CrossCorrelationWithNextRegionalDataN5Writer.class); + +} diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/zspacing/CrossCorrelationWithNextRegionalData.java b/render-ws-java-client/src/main/java/org/janelia/render/client/zspacing/CrossCorrelationWithNextRegionalData.java index 91cdeb4fd..3b4ca2af9 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/zspacing/CrossCorrelationWithNextRegionalData.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/zspacing/CrossCorrelationWithNextRegionalData.java @@ -1,6 +1,22 @@ package org.janelia.render.client.zspacing; +import java.io.IOException; +import java.io.Reader; import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.janelia.alignment.json.JsonUtils; +import org.janelia.alignment.util.FileUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Container for regional cross correlation between two adjacent layers. @@ -15,9 +31,12 @@ public class CrossCorrelationWithNextRegionalData private final Double qZ; private final String layerUrlPattern; private final Double layerCorrelation; - @SuppressWarnings("MismatchedReadAndWriteOfArray") private final double[][] regionalCorrelation; + public CrossCorrelationWithNextRegionalData() { + this(null, null, null, null, 0, 0); + } + public CrossCorrelationWithNextRegionalData(final Double pZ, final Double qZ, final String layerUrlPattern, @@ -31,6 +50,30 @@ public CrossCorrelationWithNextRegionalData(final Double pZ, this.regionalCorrelation = new double[numberOfRegionRows][numberOfRegionColumns]; } + public Double getpZ() { + return pZ; + } + + public Double getqZ() { + return qZ; + } + + public Double getLayerCorrelation() { + return layerCorrelation; + } + + public double[][] getRegionalCorrelation() { + return regionalCorrelation; + } + + public int getRegionalRowCount() { + return regionalCorrelation.length; + } + + public int getRegionalColumnCount() { + return regionalCorrelation.length == 0 ? 0 : regionalCorrelation[0].length; + } + public void setValue(final int row, final int column, final double value) { @@ -38,4 +81,58 @@ public void setValue(final int row, } public static final String DEFAULT_DATA_FILE_NAME = "poor_cc_regional_data.json.gz"; + + public static List loadDataFile(final Path path) { + final List list; + try (final Reader reader = FileUtil.DEFAULT_INSTANCE.getExtensionBasedReader(path.toString())) { + list = fromJsonArray(reader); + } catch (final Exception e) { + throw new RuntimeException("failed to load data from " + path, e); + } + + LOG.info("loadDataFile: loaded data for {} layers from {}", + list.size(), path); + + return list; + } + + public static List loadDataDirectory(final Path rootPath) + throws IOException { + + LOG.info("loadDataDirectory: entry, rootPath={}", rootPath); + + final List pathList; + try (final Stream stream = Files.walk(rootPath)) { + pathList = stream.map(Path::normalize) + .filter(path -> Files.isRegularFile(path) && DEFAULT_DATA_FILE_NAME.equals(path.toFile().getName())) + .collect(Collectors.toList()); + } + + LOG.info("loadDataDirectory: found {} {} files", pathList.size(), DEFAULT_DATA_FILE_NAME); + + final Map pZToDataMap = new HashMap<>(); + for (final Path path : pathList) { + for (final CrossCorrelationWithNextRegionalData data : loadDataFile(path)) { + pZToDataMap.put(data.pZ, data); + } + } + + LOG.info("loadDataDirectory: returning data for {} z layers", pZToDataMap.size()); + + return pZToDataMap.values().stream().sorted(Comparator.comparing(d -> d.pZ)).collect(Collectors.toList()); + } + + public static List fromJsonArray(final Reader reader) { + try { + return Arrays.asList(JsonUtils.MAPPER.readValue(reader, CrossCorrelationWithNextRegionalData[].class)); + } catch (final IOException e) { + throw new IllegalArgumentException(e); + } + } + + private static final Logger LOG = LoggerFactory.getLogger(CrossCorrelationWithNextRegionalData.class); + + private static final JsonUtils.Helper JSON_HELPER = + new JsonUtils.Helper<>(CrossCorrelationWithNextRegionalData.class); + } diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/zspacing/ZPositionCorrectionClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/zspacing/ZPositionCorrectionClientTest.java index 842b53857..de24491ba 100644 --- a/render-ws-java-client/src/test/java/org/janelia/render/client/zspacing/ZPositionCorrectionClientTest.java +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/zspacing/ZPositionCorrectionClientTest.java @@ -89,24 +89,26 @@ public static void main(final String[] args) { final String userHome = System.getProperty("user.home"); final String[] effectiveArgs = (args != null) && (args.length > 0) ? args : new String[]{ "--baseDataUrl", "http://renderer-dev.int.janelia.org:8080/render-ws/v1", - "--owner", "fibsem", - "--project", "Z0422_17_VNC_1", - "--stack", "v6_acquire_trimmed_align_destreak_1_to_1b_test_a", - "--scale", "0.22", - "--minZ", "59415", - "--maxZ", "59417", - "--runName", "testPoorCorrelation", + "--owner", "hess_wafer_53", + "--project", "cut_000_to_009", + "--stack", "c000_s095_v01_align", + "--scale", "0.05", + "--minZ", "1", + "--maxZ", "2", +// "--runName", "testWafer53", // "--runName", "testBatch", - "--correlationBatch", "1:1", +// "--correlationBatch", "1:1", // "--solveExisting", "--poorCorrelationThreshold", "0.975", + "--poorCorrelationRegionRows", "12", + "--poorCorrelationRegionColumns", "12", // "--minX", "2500", // "--maxX", "5500", // "--minY", "400", // "--maxY", "1700", // "--debugFormat", "png", -// "--optionsJson", userHome + "/Desktop/inference-options.json", + "--optionsJson", userHome + "/Desktop/zcorr/inference-options.sf_0_1.json", "--rootDirectory", userHome + "/Desktop/zcorr" }; diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java index 1794d2275..17b6a7e10 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java @@ -25,6 +25,7 @@ import org.janelia.alignment.util.Grid; import org.janelia.alignment.util.ImageProcessorCache; import org.janelia.alignment.util.ImageProcessorCacheSpec; +import org.janelia.alignment.util.NeuroglancerAttributes; import org.janelia.render.client.ClientRunner; import org.janelia.render.client.RenderDataClient; import org.janelia.render.client.parameter.CommandLineParameters; diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/n5/N5ClientTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/n5/N5ClientTest.java index 59c9243bc..7f9f3616b 100644 --- a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/n5/N5ClientTest.java +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/n5/N5ClientTest.java @@ -21,6 +21,7 @@ import org.janelia.alignment.spec.stack.StackVersion; import org.janelia.alignment.util.FileUtil; import org.janelia.alignment.util.ImageProcessorCache; +import org.janelia.alignment.util.NeuroglancerAttributes; import org.janelia.render.client.parameter.CommandLineParameters; import org.janelia.render.client.zspacing.ThicknessCorrectionData; import org.janelia.saalfeldlab.n5.DataType;