prototype saving poor regional cross correlation data to n5 and overl…

…aying in neuroglancer

render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/NeuroglancerAttributes.java → render-app/src/main/java/org/janelia/alignment/util/NeuroglancerAttributes.java

@@ -1,4 +1,4 @@
-package org.janelia.render.client.spark.n5;
+package org.janelia.alignment.util;
 
 import java.io.IOException;
 import java.nio.file.Path;

render-ws-java-client/src/main/java/org/janelia/render/client/CrossCorrelationWithNextRegionalDataN5Writer.java

@@ -0,0 +1,183 @@
+package org.janelia.render.client;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.DoubleSummaryStatistics;
+import java.util.List;
+
+import org.janelia.alignment.spec.Bounds;
+import org.janelia.alignment.spec.stack.StackMetaData;
+import org.janelia.alignment.util.NeuroglancerAttributes;
+import org.janelia.render.client.zspacing.CrossCorrelationWithNextRegionalData;
+import org.janelia.saalfeldlab.n5.DataType;
+import org.janelia.saalfeldlab.n5.GzipCompression;
+import org.janelia.saalfeldlab.n5.N5FSWriter;
+import org.janelia.saalfeldlab.n5.N5Writer;
+import org.janelia.saalfeldlab.n5.imglib2.N5Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import net.imglib2.Cursor;
+import net.imglib2.img.array.ArrayImg;
+import net.imglib2.img.array.ArrayImgs;
+import net.imglib2.img.basictypeaccess.array.ByteArray;
+import net.imglib2.type.numeric.integer.UnsignedByteType;
+
+public class CrossCorrelationWithNextRegionalDataN5Writer
+        implements Serializable {
+
+    public static void writeN5(final List<CrossCorrelationWithNextRegionalData> dataList,
+                               final String basePath,
+                               final String datasetName,
+                               final StackMetaData stackMetaData) {
+
+        final int numberOfLayers = dataList.size();
+        LOG.info("writeN5: entry, processing {} z layers", numberOfLayers);
+
+        final CrossCorrelationWithNextRegionalData firstLayerData = dataList.get(0);
+        final int rowCount = firstLayerData.getRegionalRowCount();
+        final int columnCount = firstLayerData.getRegionalColumnCount();
+        final Bounds stackBounds = stackMetaData.getStats().getStackBounds();
+
+        if ((rowCount > 0) && (columnCount >0)) {
+
+            // need to write z layers concurrently, so make z block size 1
+            final int[] blockSize = { columnCount, rowCount, 1 };
+            final long[] blockDimensions = { blockSize[0], blockSize[1], blockSize[2] };
+            final long[] datasetDimensions = { blockSize[0], blockSize[1], (long) stackBounds.getDeltaZ() };
+
+            try (final N5Writer n5Writer = new N5FSWriter(basePath)) {
+
+                n5Writer.createDataset(datasetName,
+                                       datasetDimensions,
+                                       blockSize,
+                                       DataType.UINT8,
+                                       new GzipCompression());
+
+                final int margin = 55;
+                final int minViz = 0;
+                final int maxViz = 255 - margin;
+                final int vizRange = maxViz - minViz;
+
+                for (int i = 0; i < numberOfLayers; i++) {
+                    final CrossCorrelationWithNextRegionalData ccWithNext = dataList.get(i);
+                    final long[] gridOffset = { 0, 0, (long) (ccWithNext.getpZ() - stackBounds.getMinZ()) };
+                    final ArrayImg<UnsignedByteType, ByteArray> block = ArrayImgs.unsignedBytes(blockDimensions);
+                    final Cursor<UnsignedByteType> out = block.cursor();
+
+                    final double[][] regionalCorrelation = ccWithNext.getRegionalCorrelation();
+
+                    final DoubleSummaryStatistics nonZeroCorrelationStats =
+                            Arrays.stream(regionalCorrelation)
+                                    .flatMapToDouble(Arrays::stream)
+                                    .filter(c -> c > 0.0)
+                                    .summaryStatistics();
+
+                    final double minCc = nonZeroCorrelationStats.getMin();
+                    final double ccRange = nonZeroCorrelationStats.getMax() - minCc;
+
+                    // System.out.println("stats=" + nonZeroCorrelationStats + ", ccRange=" + ccRange + ", vizRange=" + vizRange);
+
+                    for (int row = 0; row < rowCount; row++) {
+                        for (int column = 0; column < columnCount; column++) {
+                            final UnsignedByteType outValue = out.next();
+                            final double correlation = regionalCorrelation[row][column];
+                            if (correlation > 0.0) {
+                                final double relativeCc = correlation - minCc;
+                                final double relativeViz = relativeCc * vizRange / ccRange;
+                                //final double invertedRelativeViz = vizRange - relativeViz; // make worse correlation brighter
+                                final int viz = (int) relativeViz + margin;
+                                // System.out.println("[" + outValue.getIndex() + "] => " + correlation + ", " + viz);
+                                outValue.set(viz);
+                            }
+                        }
+                    }
+
+                    N5Utils.saveNonEmptyBlock(block, n5Writer, datasetName, gridOffset, new UnsignedByteType());
+
+                    if (i % 5 == 0) {
+                        LOG.info("writeN5: saved blocks for {} out of {} z layers", (i+1), numberOfLayers);
+                    }
+                }
+
+                final List<Double> stackResolutionValues = stackMetaData.getCurrentResolutionValues();
+                final double pixelsPerColumn = stackBounds.getWidth() / (double) columnCount;
+                final double columnResolution = pixelsPerColumn * stackResolutionValues.get(0);
+                final double pixelsPerRow = stackBounds.getHeight() / (double) rowCount;
+                final double rowResolution = pixelsPerRow * stackResolutionValues.get(1);
+                final List<Double> ccVolumeResolutionValues =
+                        Arrays.asList(columnResolution, rowResolution, stackResolutionValues.get(2));
+
+                // TODO: figure out why neuroglancer ignores this for cc data n5
+                final List<Long> translationList = Arrays.asList(stackBounds.getMinX().longValue(),
+                                                                 stackBounds.getMinY().longValue(),
+                                                                 stackBounds.getMinZ().longValue());
+
+                // save additional parameters so that n5 can be viewed in neuroglancer
+                final NeuroglancerAttributes ngAttributes =
+                        new NeuroglancerAttributes(ccVolumeResolutionValues,
+                                                   "nm",
+                                                   0,
+                                                   new int[0],
+                                                   translationList,
+                                                   NeuroglancerAttributes.NumpyContiguousOrdering.FORTRAN);
+
+                ngAttributes.write(Paths.get(basePath),
+                                   Paths.get(datasetName));
+
+            } catch (final IOException e) {
+                throw new RuntimeException(e);
+            }
+
+        }
+
+        LOG.info("writeN5: exit");
+    }
+
+    @SuppressWarnings("CommentedOutCode")
+    public static void main(final String[] args) {
+/*
+        try {
+//            final String baseOutputPath = "/Users/trautmane/Desktop/test-ic.n5";
+//            final String zCorrDir = "/Users/trautmane/Desktop/zcorr";
+//            final String owner = "hess_wafer_53";
+//            final String project = "cut_000_to_009";
+//            final String stack = "c000_s095_v01_align";
+//            final String run = "run_20230726_161543";
+
+
+            final String baseOutputPath = "/nrs/cellmap/data/jrc_ut23-0590-001/jrc_ut23-0590-001.n5";
+            final String zCorrDir = "/nrs/cellmap/data/jrc_ut23-0590-001/z_corr";
+            final String owner = "cellmap";
+            final String project = "jrc_ut23_0590_001";
+            final String stack = "v1_acquire_align";
+            final String run = "run_20230803_092548_69_z_corr";
+
+            final Path runPath = Paths.get(zCorrDir, owner, project, stack, run);
+            final List<CrossCorrelationWithNextRegionalData> dataList =
+                    CrossCorrelationWithNextRegionalData.loadDataDirectory(runPath);
+
+            final RenderDataClient renderDataClient =
+                    new RenderDataClient("http://renderer-dev.int.janelia.org:8080/render-ws/v1",
+                                         owner,
+                                         project);
+            final StackMetaData stackMetaData = renderDataClient.getStackMetaData(stack);
+            final String datasetName = "/cc/regional/" + project + "/" + stack + "/" + run + "/s0";
+
+            writeN5(dataList,
+                    baseOutputPath,
+                    datasetName,
+                    stackMetaData);
+        } catch (final Throwable t) {
+            t.printStackTrace();
+        }
+
+*/
+    }
+
+    private static final Logger LOG = LoggerFactory.getLogger(CrossCorrelationWithNextRegionalDataN5Writer.class);
+
+}

render-ws-java-client/src/main/java/org/janelia/render/client/zspacing/CrossCorrelationWithNextRegionalData.java

@@ -1,6 +1,22 @@
 package org.janelia.render.client.zspacing;
 
+import java.io.IOException;
+import java.io.Reader;
 import java.io.Serializable;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.janelia.alignment.json.JsonUtils;
+import org.janelia.alignment.util.FileUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Container for regional cross correlation between two adjacent layers.
@@ -15,9 +31,12 @@ public class CrossCorrelationWithNextRegionalData
     private final Double qZ;
     private final String layerUrlPattern;
     private final Double layerCorrelation;
-    @SuppressWarnings("MismatchedReadAndWriteOfArray")
     private final double[][] regionalCorrelation;
 
+    public CrossCorrelationWithNextRegionalData() {
+        this(null, null, null, null, 0, 0);
+    }
+
     public CrossCorrelationWithNextRegionalData(final Double pZ,
                                                 final Double qZ,
                                                 final String layerUrlPattern,
@@ -31,11 +50,89 @@ public CrossCorrelationWithNextRegionalData(final Double pZ,
         this.regionalCorrelation = new double[numberOfRegionRows][numberOfRegionColumns];
     }
 
+    public Double getpZ() {
+        return pZ;
+    }
+
+    public Double getqZ() {
+        return qZ;
+    }
+
+    public Double getLayerCorrelation() {
+        return layerCorrelation;
+    }
+
+    public double[][] getRegionalCorrelation() {
+        return regionalCorrelation;
+    }
+
+    public int getRegionalRowCount() {
+        return regionalCorrelation.length;
+    }
+
+    public int getRegionalColumnCount() {
+        return regionalCorrelation.length == 0 ? 0 : regionalCorrelation[0].length;
+    }
+
     public void setValue(final int row,
                          final int column,
                          final double value) {
         regionalCorrelation[row][column] = value;
     }
 
     public static final String DEFAULT_DATA_FILE_NAME = "poor_cc_regional_data.json.gz";
+
+    public static List<CrossCorrelationWithNextRegionalData> loadDataFile(final Path path) {
+        final List<CrossCorrelationWithNextRegionalData> list;
+        try (final Reader reader = FileUtil.DEFAULT_INSTANCE.getExtensionBasedReader(path.toString())) {
+            list = fromJsonArray(reader);
+        } catch (final Exception e) {
+            throw new RuntimeException("failed to load data from " + path, e);
+        }
+
+        LOG.info("loadDataFile: loaded data for {} layers from {}",
+                 list.size(), path);
+
+        return list;
+    }
+
+    public static List<CrossCorrelationWithNextRegionalData> loadDataDirectory(final Path rootPath)
+            throws IOException {
+
+        LOG.info("loadDataDirectory: entry, rootPath={}", rootPath);
+
+        final List<Path> pathList;
+        try (final Stream<Path> stream = Files.walk(rootPath)) {
+            pathList = stream.map(Path::normalize)
+                    .filter(path -> Files.isRegularFile(path) && DEFAULT_DATA_FILE_NAME.equals(path.toFile().getName()))
+                    .collect(Collectors.toList());
+        }
+
+        LOG.info("loadDataDirectory: found {} {} files", pathList.size(), DEFAULT_DATA_FILE_NAME);
+
+        final Map<Double, CrossCorrelationWithNextRegionalData> pZToDataMap = new HashMap<>();
+        for (final Path path : pathList) {
+            for (final CrossCorrelationWithNextRegionalData data : loadDataFile(path)) {
+                pZToDataMap.put(data.pZ, data);
+            }
+        }
+
+        LOG.info("loadDataDirectory: returning data for {} z layers", pZToDataMap.size());
+
+        return pZToDataMap.values().stream().sorted(Comparator.comparing(d -> d.pZ)).collect(Collectors.toList());
+    }
+
+    public static List<CrossCorrelationWithNextRegionalData> fromJsonArray(final Reader reader) {
+        try {
+            return Arrays.asList(JsonUtils.MAPPER.readValue(reader, CrossCorrelationWithNextRegionalData[].class));
+        } catch (final IOException e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+
+    private static final Logger LOG = LoggerFactory.getLogger(CrossCorrelationWithNextRegionalData.class);
+
+    private static final JsonUtils.Helper<CrossCorrelationWithNextRegionalData> JSON_HELPER =
+            new JsonUtils.Helper<>(CrossCorrelationWithNextRegionalData.class);
+
 }

render-ws-java-client/src/test/java/org/janelia/render/client/zspacing/ZPositionCorrectionClientTest.java

@@ -89,24 +89,26 @@ public static void main(final String[] args) {
         final String userHome = System.getProperty("user.home");
         final String[] effectiveArgs = (args != null) && (args.length > 0) ? args : new String[]{
                 "--baseDataUrl", "http://renderer-dev.int.janelia.org:8080/render-ws/v1",
-                "--owner", "fibsem",
-                "--project", "Z0422_17_VNC_1",
-                "--stack", "v6_acquire_trimmed_align_destreak_1_to_1b_test_a",
-                "--scale", "0.22",
-                "--minZ", "59415",
-                "--maxZ", "59417",
-                "--runName", "testPoorCorrelation",
+                "--owner", "hess_wafer_53",
+                "--project", "cut_000_to_009",
+                "--stack", "c000_s095_v01_align",
+                "--scale", "0.05",
+                "--minZ", "1",
+                "--maxZ", "2",
+//                "--runName", "testWafer53",
 //                "--runName", "testBatch",
-                "--correlationBatch", "1:1",
+//                "--correlationBatch", "1:1",
 //                "--solveExisting",
                 "--poorCorrelationThreshold", "0.975",
+                "--poorCorrelationRegionRows", "12",
+                "--poorCorrelationRegionColumns", "12",
 
 //                "--minX", "2500",
 //                "--maxX", "5500",
 //                "--minY", "400",
 //                "--maxY", "1700",
 //                "--debugFormat", "png",
-//                "--optionsJson", userHome + "/Desktop/inference-options.json",
+                "--optionsJson", userHome + "/Desktop/zcorr/inference-options.sf_0_1.json",
                 "--rootDirectory", userHome + "/Desktop/zcorr"
         };
 

render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java

@@ -25,6 +25,7 @@
 import org.janelia.alignment.util.Grid;
 import org.janelia.alignment.util.ImageProcessorCache;
 import org.janelia.alignment.util.ImageProcessorCacheSpec;
+import org.janelia.alignment.util.NeuroglancerAttributes;
 import org.janelia.render.client.ClientRunner;
 import org.janelia.render.client.RenderDataClient;
 import org.janelia.render.client.parameter.CommandLineParameters;

render-ws-spark-client/src/test/java/org/janelia/render/client/spark/n5/N5ClientTest.java

@@ -21,6 +21,7 @@
 import org.janelia.alignment.spec.stack.StackVersion;
 import org.janelia.alignment.util.FileUtil;
 import org.janelia.alignment.util.ImageProcessorCache;
+import org.janelia.alignment.util.NeuroglancerAttributes;
 import org.janelia.render.client.parameter.CommandLineParameters;
 import org.janelia.render.client.zspacing.ThicknessCorrectionData;
 import org.janelia.saalfeldlab.n5.DataType;