From 9f3399ce082810929d04cac9c9bc66331616b440 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Sat, 20 Apr 2024 16:57:52 -0400 Subject: [PATCH 01/16] Add first version of visualization client --- .../client/FlatfieldVisualizationClient.java | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 render-ws-java-client/src/main/java/org/janelia/render/client/FlatfieldVisualizationClient.java diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/FlatfieldVisualizationClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/FlatfieldVisualizationClient.java new file mode 100644 index 000000000..b0dd6be31 --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/FlatfieldVisualizationClient.java @@ -0,0 +1,136 @@ +package org.janelia.render.client; + +import org.janelia.alignment.ImageAndMask; +import org.janelia.alignment.spec.ChannelSpec; +import org.janelia.alignment.spec.LayoutData; +import org.janelia.alignment.spec.ResolvedTileSpecCollection; +import org.janelia.alignment.spec.TileSpec; +import org.janelia.alignment.spec.stack.MipmapPathBuilder; +import org.janelia.alignment.spec.stack.StackMetaData; +import org.janelia.render.client.parameter.TileSpecValidatorParameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.awt.*; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +/** + * This is a one-off client for visualizing the estimated flat field of the Hess Lab's wafer_53 + */ +public class FlatfieldVisualizationClient { + // the path to the data (converted to 8-bit PNGs; must be on a shared filesystem accessible to the web server) + private static final String DATA_PATH = "/nrs/hess/render/flatfield/flatfields_n2000"; + // the format for the files (fill in z and sfov number for the placeholders, in order) + private static final String NAME_FORMAT = "flat_field_z%03d_sfov%03d_n2000.png"; + // the number of z slices + private static final int Z_SLICES = 47; + + // stack location + private static final String BASE_URL = "http://renderer-dev.int.janelia.org:8080/render-ws/v1"; + private static final String OWNER = "hess_wafer_53d"; + private static final String PROJECT = "slab_000_to_009"; + private static final String STACK = "flatfield_test1"; + // reuse metadata from random stack in same project to set up new stack + private static final String TEMPLATE_STACK = "s000_m209"; + + public static void main(final String[] args) throws IOException { + LOG.info("main: setting up target stack {}", STACK); + final RenderDataClient dataClient = new RenderDataClient(BASE_URL, OWNER, PROJECT); + final StackMetaData projectSpecificMetaData = dataClient.getStackMetaData(TEMPLATE_STACK); + dataClient.setupDerivedStack(projectSpecificMetaData, STACK); + + final Map tileSpecTemplate = getTemplateLayer(dataClient); + for (int z = 1; z <= Z_SLICES; z++) { + final ResolvedTileSpecCollection slice = assembleSlice(tileSpecTemplate, z); + dataClient.saveResolvedTiles(slice, STACK, (double) z); + } + + LOG.info("main: completing target stack {}", STACK); + dataClient.setStackState(STACK, StackMetaData.StackState.COMPLETE); + } + + private static Map getTemplateLayer(final RenderDataClient dataClient) throws IOException { + LOG.info("getTemplateLayer: getting template layer from stack {}", TEMPLATE_STACK); + final ResolvedTileSpecCollection realTileSpecs = dataClient.getResolvedTiles(TEMPLATE_STACK, 1.0, ".*_000001_.*"); + + final Map templateTileSpecs = new HashMap<>(); + final int rowOffset = 10; + + final TileSpec firstTileSpec = realTileSpecs.getTileSpecs().stream().findFirst().orElseThrow(); + final double height = firstTileSpec.getHeight(); + final double width = firstTileSpec.getWidth(); + + for (final TileSpec tileSpec : realTileSpecs.getTileSpecs()) { + final TileSpec templateTileSpec = new TileSpec(); + + // set layout + final LayoutData originalLayout = tileSpec.getLayout(); + final LayoutData layoutData = new LayoutData( + "1.0", + originalLayout.getTemca(), + originalLayout.getCamera(), + originalLayout.getImageRow() - rowOffset, + originalLayout.getImageCol(), + originalLayout.getImageCol() * width, + (originalLayout.getImageRow() - rowOffset) * height, + 0.0 + ); + templateTileSpec.setLayout(layoutData); + + // set bounds + final Rectangle bounds = new Rectangle(templateTileSpec.getLayout().getStageX().intValue(), templateTileSpec.getLayout().getStageY().intValue(), (int) width, (int) height); + templateTileSpec.setBoundingBox(bounds, 2000.0); + templateTileSpec.setWidth(width); + templateTileSpec.setHeight(height); + + // insert at correct position + final int sfovNumber = getSfovNumber(tileSpec.getTileId()); + templateTileSpecs.put(sfovNumber, templateTileSpec); + } + + return templateTileSpecs; + } + + private static int getSfovNumber(final String tileId) { + final String[] parts = tileId.split("_"); + return Integer.parseInt(parts[2]); + } + + private static ResolvedTileSpecCollection assembleSlice(final Map templateTileSpecs, final int z) { + LOG.info("assembleSlice: assembling slice {}", z); + final ResolvedTileSpecCollection slice = new ResolvedTileSpecCollection(); + for (final Map.Entry entry : templateTileSpecs.entrySet()) { + final int sfov = entry.getKey(); + final TileSpec template = entry.getValue(); + + // set metadata + final TileSpec tileSpec = new TileSpec(); + tileSpec.setTileId(String.format("z%03d_sfov%03d", z, sfov)); + tileSpec.setLayout(template.getLayout()); + tileSpec.setBoundingBox(template.toTileBounds().toRectangle(), 2000.0); + tileSpec.setWidth((double) template.getWidth()); + tileSpec.setHeight((double) template.getHeight()); + tileSpec.setZ((double) z); + + // set image + final String imagePath = DATA_PATH + "/" + String.format(NAME_FORMAT, z, sfov); + final ChannelSpec channelSpec = createChannelSpec(imagePath); + tileSpec.addChannel(channelSpec); + tileSpec.getFirstMipmapEntry(); + slice.addTileSpecToCollection(tileSpec); + } + return slice; + } + + private static ChannelSpec createChannelSpec(String imagePath) { + final MipmapPathBuilder mipmapPathBuilder = new MipmapPathBuilder(imagePath, 0, ".png", null); + final TreeMap mipmapLevels = new TreeMap<>(); + mipmapLevels.put(0, new ImageAndMask(imagePath, null)); + return new ChannelSpec("default", null, null, mipmapLevels, mipmapPathBuilder, null); + } + + private static final Logger LOG = LoggerFactory.getLogger(FlatfieldVisualizationClient.class); +} From d5b5e326b69a97b11546c59ca4e59546a3346fef Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Sat, 20 Apr 2024 17:02:30 -0400 Subject: [PATCH 02/16] Fix wrong positioning --- .../janelia/render/client/FlatfieldVisualizationClient.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/FlatfieldVisualizationClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/FlatfieldVisualizationClient.java index b0dd6be31..60be606ae 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/FlatfieldVisualizationClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/FlatfieldVisualizationClient.java @@ -7,7 +7,6 @@ import org.janelia.alignment.spec.TileSpec; import org.janelia.alignment.spec.stack.MipmapPathBuilder; import org.janelia.alignment.spec.stack.StackMetaData; -import org.janelia.render.client.parameter.TileSpecValidatorParameters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,7 +73,7 @@ private static Map getTemplateLayer(final RenderDataClient da originalLayout.getCamera(), originalLayout.getImageRow() - rowOffset, originalLayout.getImageCol(), - originalLayout.getImageCol() * width, + originalLayout.getImageCol() * width / 2, (originalLayout.getImageRow() - rowOffset) * height, 0.0 ); From 2341ea2ee0a639bc74dd2066065e6d62aef2aa21 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Sat, 20 Apr 2024 18:39:16 -0400 Subject: [PATCH 03/16] Fix a few errors and add transformation --- .../client/FlatfieldVisualizationClient.java | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/FlatfieldVisualizationClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/FlatfieldVisualizationClient.java index 60be606ae..565ab08cb 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/FlatfieldVisualizationClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/FlatfieldVisualizationClient.java @@ -3,16 +3,18 @@ import org.janelia.alignment.ImageAndMask; import org.janelia.alignment.spec.ChannelSpec; import org.janelia.alignment.spec.LayoutData; +import org.janelia.alignment.spec.LeafTransformSpec; import org.janelia.alignment.spec.ResolvedTileSpecCollection; import org.janelia.alignment.spec.TileSpec; -import org.janelia.alignment.spec.stack.MipmapPathBuilder; +import org.janelia.alignment.spec.TransformSpec; import org.janelia.alignment.spec.stack.StackMetaData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.awt.*; +import java.awt.Rectangle; import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -21,7 +23,7 @@ */ public class FlatfieldVisualizationClient { // the path to the data (converted to 8-bit PNGs; must be on a shared filesystem accessible to the web server) - private static final String DATA_PATH = "/nrs/hess/render/flatfield/flatfields_n2000"; + private static final String DATA_PATH = "/nrs/hess/render/flatfield/flatfields_n2000_8bit"; // the format for the files (fill in z and sfov number for the placeholders, in order) private static final String NAME_FORMAT = "flat_field_z%03d_sfov%03d_n2000.png"; // the number of z slices @@ -31,7 +33,7 @@ public class FlatfieldVisualizationClient { private static final String BASE_URL = "http://renderer-dev.int.janelia.org:8080/render-ws/v1"; private static final String OWNER = "hess_wafer_53d"; private static final String PROJECT = "slab_000_to_009"; - private static final String STACK = "flatfield_test1"; + private static final String STACK = "flatfield_estimate"; // reuse metadata from random stack in same project to set up new stack private static final String TEMPLATE_STACK = "s000_m209"; @@ -114,21 +116,24 @@ private static ResolvedTileSpecCollection assembleSlice(final Map mipmapLevels = new TreeMap<>(); mipmapLevels.put(0, new ImageAndMask(imagePath, null)); - return new ChannelSpec("default", null, null, mipmapLevels, mipmapPathBuilder, null); + return new ChannelSpec("default", null, null, mipmapLevels, null, null); } private static final Logger LOG = LoggerFactory.getLogger(FlatfieldVisualizationClient.class); From ef55898e2bf1017f40f9391fcfd28fd5cee28a70 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Mon, 22 Apr 2024 13:56:15 -0400 Subject: [PATCH 04/16] Add first version of filter that does the processing --- .../filter/ImageCalculatorFilter.java | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 render-app/src/main/java/org/janelia/alignment/filter/ImageCalculatorFilter.java diff --git a/render-app/src/main/java/org/janelia/alignment/filter/ImageCalculatorFilter.java b/render-app/src/main/java/org/janelia/alignment/filter/ImageCalculatorFilter.java new file mode 100644 index 000000000..70fb5d7bd --- /dev/null +++ b/render-app/src/main/java/org/janelia/alignment/filter/ImageCalculatorFilter.java @@ -0,0 +1,99 @@ +package org.janelia.alignment.filter; + +import ij.ImagePlus; +import ij.process.ImageProcessor; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.function.DoubleBinaryOperator; + +public class ImageCalculatorFilter implements Filter { + + public enum BinaryOperation { + ADD(Double::sum), + SUBTRACT((a, b) -> a - b), + MULTIPLY((a, b) -> a * b), + DIVIDE((a, b) -> a / b), + MIN(Math::min), + MAX(Math::max), + AVERAGE((a, b) -> (a + b) / 2.0), + COPY((a, b) -> b), + DIFFERENCE((a, b) -> Math.abs(a - b)); + + private final DoubleBinaryOperator operator; + + BinaryOperation(final DoubleBinaryOperator operator) { + this.operator = operator; + } + + public double apply(final double a, final double b) { + return operator.applyAsDouble(a, b); + } + + public static BinaryOperation fromString(final String operation) { + for (final BinaryOperation op : values()) { + if (op.name().equalsIgnoreCase(operation)) { + return op; + } + } + throw new IllegalArgumentException("unknown operation: " + operation); + } + } + + private String secondImageUri; + private ImageProcessor secondImage; + private BinaryOperation operation; + + // empty constructor required to create instances from specifications + @SuppressWarnings("unused") + public ImageCalculatorFilter() { + this(null, null); + } + + public ImageCalculatorFilter(final String secondImageUri, final BinaryOperation operation) { + this.secondImageUri = secondImageUri; + this.operation = operation; + initializeSecondImage(); + } + + @Override + public void init(final Map params) { + this.operation = BinaryOperation.fromString(Filter.getStringParameter("operation", params)); + this.secondImageUri = Filter.getStringParameter("secondImageUri", params); + initializeSecondImage(); + } + + private void initializeSecondImage() { + if (secondImageUri == null || operation == null) { + throw new IllegalStateException("secondImageUri and operator must be set before initializing the second image"); + } + final ImagePlus imp = new ImagePlus(secondImageUri); + secondImage = imp.getProcessor().convertToFloat(); + } + + @Override + public Map toParametersMap() { + final Map map = new LinkedHashMap<>(); + map.put("operation", operation.name()); + map.put("secondImageUri", secondImageUri); + return map; + } + + @Override + public void process(final ImageProcessor ip, final double scale) { + // TODO: check if not allowing scaling makes sense + if (scale != 1.0) { + throw new IllegalArgumentException("scaling is not supported for this filter"); + } + + // convert to 32-bit grayscale (float) for lossless processing + final ImageProcessor firstImage = ip.convertToFloat(); + + // apply the operation + for (int i = 0; i < ip.getPixelCount(); i++) { + final double a = firstImage.getf(i); + final double b = secondImage.getf(i); + ip.setf(i, (float) operation.apply(a, b)); + } + } +} From 4272ce8c300c02da04e6e5f46078c704600c57a1 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Tue, 23 Apr 2024 13:41:22 -0400 Subject: [PATCH 05/16] Fix wrong comment in SmoothMaskStreakCorrector --- .../janelia/alignment/destreak/SmoothMaskStreakCorrector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/render-app/src/main/java/org/janelia/alignment/destreak/SmoothMaskStreakCorrector.java b/render-app/src/main/java/org/janelia/alignment/destreak/SmoothMaskStreakCorrector.java index a076cd2df..478f6e035 100644 --- a/render-app/src/main/java/org/janelia/alignment/destreak/SmoothMaskStreakCorrector.java +++ b/render-app/src/main/java/org/janelia/alignment/destreak/SmoothMaskStreakCorrector.java @@ -26,7 +26,7 @@ * The tunable parameters are: *
    *
  • innerCutoff: inner radius of the cutoff profile in px; smaller values may improve de-streaking, values that are too small tinker with the overall intensity of the image
    • - *
  • bandWidth: width of the gaussian band in px; smaller values may improve de-streaking, values that are too high blur the image in the direction orthogonal to the angle
    • + *
  • bandWidth: width of the gaussian band in px; larger values may improve de-streaking, values that are too high blur the image in the direction orthogonal to the angle
    • *
  • angle: angle of the band in deg; this should be orthogonal to the streaks (e.g., choose an angle of 0.0 for vertical streaks and 90.0 vor horizontal ones)
    • *
* From bf76613e5406529ac91d9919acccc47f9c8ea87f Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Tue, 23 Apr 2024 16:59:12 -0400 Subject: [PATCH 06/16] Add client to re-save flat-field corrected images --- .../MultiSemFlatFieldCorrectionClient.java | 194 ++++++++++++++++++ ...MultiSemFlatFieldCorrectionClientTest.java | 34 +++ 2 files changed, 228 insertions(+) create mode 100644 render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java create mode 100644 render-ws-java-client/src/test/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClientTest.java diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java new file mode 100644 index 000000000..ac6e2a393 --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java @@ -0,0 +1,194 @@ +package org.janelia.render.client; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.ParametersDelegate; +import ij.IJ; +import ij.ImagePlus; +import ij.process.FloatProcessor; +import ij.process.ImageProcessor; +import org.janelia.alignment.ImageAndMask; +import org.janelia.alignment.loader.ImageLoader; +import org.janelia.alignment.spec.ChannelSpec; +import org.janelia.alignment.spec.ResolvedTileSpecCollection; +import org.janelia.alignment.spec.TileSpec; +import org.janelia.alignment.spec.stack.StackId; +import org.janelia.alignment.spec.stack.StackMetaData; +import org.janelia.alignment.util.ImageProcessorCache; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MultiProjectParameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.DoubleSummaryStatistics; +import java.util.List; + +/** + * Apply a previously computed flat field estimate to correct shading in a Multi-SEM project. + * Originally developed for the wafer_53 Multi-SEM dataset. + * + * @author Michael Innerberger + */ +public class MultiSemFlatFieldCorrectionClient { + + // make cache large enough to hold all flat field estimates for one layer + private static final ImageProcessorCache CACHE = new ImageProcessorCache(91L * 2000L * 1748L + 1, false, false); + + private final Parameters params; + private final RenderDataClient renderClient; + + public static class Parameters extends CommandLineParameters { + @ParametersDelegate + private final MultiProjectParameters multiProject = new MultiProjectParameters(); + @Parameter(names = "--flatFieldLocation", description = "Location of the flat field estimates", required = true) + private String flatFieldLocation; + @Parameter(names = "--outputRoot", description = "Folder to write corrected images to", required = true) + private String outputFolder; + @Parameter(names = "--flatFieldFormat", description = "Format of the flat field estimates in printf style with placeholders for z-layer and sfov", required = true) + private String flatFieldFormat; + @Parameter(names = "--targetStackSuffix", description = "Suffix to append to the stack name for the corrected stack", required = true) + private String targetStackSuffix = "_corrected"; + @Parameter(names = "--inputRoot", description = "Root folder for input data; if given, the structure under this root is replicated in the output folder") + private String inputRoot = null; + } + + public static void main(final String[] args) { + final ClientRunner clientRunner = new ClientRunner(args) { + @Override + public void runClient(final String[] args) throws IOException { + + final Parameters parameters = new Parameters(); + parameters.parse(args); + LOG.info("runClient: entry, parameters={}", parameters); + + final MultiSemFlatFieldCorrectionClient client = new MultiSemFlatFieldCorrectionClient(parameters); + client.correctTiles(); + } + }; + clientRunner.run(); + } + + public MultiSemFlatFieldCorrectionClient(final Parameters parameters) { + this.params = parameters; + this.renderClient = new RenderDataClient(parameters.multiProject.baseDataUrl, parameters.multiProject.owner, parameters.multiProject.project); + } + + public void correctTiles() throws IOException { + final List stacks = params.multiProject.stackIdWithZ.getStackIdList(renderClient); + + // TODO: iterate by z-layer first to re-use the flat field estimate for all stacks? + for (final StackId stack : stacks) { + final DoubleSummaryStatistics zStatistics = getZStatistics(stack); + final StackMetaData stackMetaData = renderClient.getStackMetaData(stack.getStack()); + renderClient.setupDerivedStack(stackMetaData, stack.getStack() + params.targetStackSuffix); + +// for (int z = (int) zStatistics.getMin(); z <= (int) zStatistics.getMax(); z++) { + for (int z = 1; z <= 1; z++) { + final ResolvedTileSpecCollection tileSpecs = renderClient.getResolvedTiles(stack.getStack(), (double) z); + + for (final TileSpec tileSpec : tileSpecs.getTileSpecs()) { + final ImageProcessor ip = loadImageTile(tileSpec); + final int sfov = extractSfovNumber(tileSpec); + final ImageProcessor flatFieldEstimate = loadFlatFieldEstimate(z, sfov); + + applyFlatFieldCorrection(ip, flatFieldEstimate); + + patchTileSpec(tileSpec); + saveImage(ip, tileSpec); + } + + renderClient.saveResolvedTiles(tileSpecs, stack.getStack() + params.targetStackSuffix, (double) z); + } + renderClient.setStackState(stack.getStack() + params.targetStackSuffix, StackMetaData.StackState.COMPLETE); + } + } + + private DoubleSummaryStatistics getZStatistics(final StackId stack) throws IOException { + final List zValues = renderClient.getStackZValues(stack.getStack()); + final DoubleSummaryStatistics zStatistics = zValues.stream().mapToDouble(Double::doubleValue).summaryStatistics(); + LOG.info("Considering {} with {} z-layers", stack, zStatistics.getCount()); + return zStatistics; + } + + private ImageProcessor loadFlatFieldEstimate(final int z, final int sfov) { + final Path imagePath = Path.of(params.flatFieldLocation, String.format(params.flatFieldFormat, z, sfov)); + final String imageUrl = "file:" + imagePath; + final ImageLoader.LoaderType loaderType = ImageLoader.LoaderType.IMAGEJ_DEFAULT; + return CACHE.get(imageUrl, 0, false, false, loaderType, null); + } + + private int extractSfovNumber(final TileSpec tileSpec) { + final String tileId = tileSpec.getTileId(); + final String[] parts = tileId.split("_"); + return Integer.parseInt(parts[1].substring(4)); + } + + /** + * Apply the flat field estimate to the input image. + * @param ip the input image that is altered in place + * @param flatFieldEstimate the flat field estimate to apply + */ + private void applyFlatFieldCorrection(final ImageProcessor ip, final ImageProcessor flatFieldEstimate) { + // convert to 32-bit grayscale (float) for lossless processing + final FloatProcessor fp = ip.convertToFloatProcessor(); + + for (int i = 0; i < ip.getPixelCount(); i++) { + final double a = fp.getf(i); + final double b = flatFieldEstimate.getf(i); + fp.setf(i, (float) (a / b)); + } + + // convert back to original bit depth + fp.setMinAndMax(0, 255); + ip.setPixels(0, fp); + } + + private void patchTileSpec(final TileSpec tileSpec) { + final Path originalPath = Path.of(tileSpec.getTileImageUrl().replaceFirst("file:", "")); + final Path newPath; + + if (params.inputRoot != null) { + final Path relativePath = Path.of(params.inputRoot).relativize(originalPath); + newPath = Path.of(params.outputFolder).resolve(relativePath); + } else { + newPath = Path.of(params.outputFolder).resolve(originalPath.getFileName()); + } + + ensureFolderExists(newPath.getParent()); + final ChannelSpec firstChannel = tileSpec.getAllChannels().get(0); + final ImageAndMask originalImage = firstChannel.getFirstMipmapEntry().getValue(); + final ImageAndMask newImage = originalImage.copyWithImage(newPath.toString(), null, null); + firstChannel.putMipmap(0, newImage); + } + + private void ensureFolderExists(final Path folder) { + final boolean folderExists = folder.toFile().exists() || folder.toFile().mkdirs(); + + if (!folderExists) { + LOG.error("Could not create output folder: {}", folder); + System.exit(1); + } + } + + private ImageProcessor loadImageTile(final TileSpec tileSpec) { + final ChannelSpec firstChannelSpec = tileSpec.getAllChannels().get(0); + final String tileId = tileSpec.getTileId(); + final ImageAndMask imageAndMask = firstChannelSpec.getFirstMipmapImageAndMask(tileId); + + return ImageProcessorCache.DISABLED_CACHE.get(imageAndMask.getImageUrl(), + 0, + false, + firstChannelSpec.is16Bit(), + imageAndMask.getImageLoaderType(), + imageAndMask.getImageSliceNumber()); + } + + private void saveImage(final ImageProcessor ip, final TileSpec tileSpec) { + final String tileId = tileSpec.getTileId(); + final ImagePlus imp = new ImagePlus(tileId, ip); + IJ.save(imp, tileSpec.getImagePath()); + } + + private static final Logger LOG = LoggerFactory.getLogger(MultiSemFlatFieldCorrectionClient.class); +} diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClientTest.java new file mode 100644 index 000000000..3d78a90a9 --- /dev/null +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClientTest.java @@ -0,0 +1,34 @@ +package org.janelia.render.client; + +import org.janelia.render.client.parameter.CommandLineParameters; +import org.junit.Test; + +/** + * Tests the {@link MultiSemFlatFieldCorrectionClient} class. + * + * @author Michael Innerberger + */ +public class MultiSemFlatFieldCorrectionClientTest { + + @Test + public void testParameterParsing() throws Exception { + CommandLineParameters.parseHelp(new MultiSemFlatFieldCorrectionClient.Parameters()); + } + + public static void main(String[] args) { + args = new String[] { + "--baseDataUrl", "http://renderer-dev.int.janelia.org:8080/render-ws/v1", + "--owner", "hess_wafer_53d", + "--project", "slab_000_to_009", + "--stackPattern", "s000_.*", + "--targetStackSuffix", "_corrected", + "--inputRoot", "/nrs/hess/data/hess_wafer_53/raw/imaging/msem", + "--outputRoot", "/nrs/hess/render/flatfield/test/data", + "--flatFieldLocation", "/nrs/hess/render/flatfield/flatfields_n2000", + "--flatFieldFormat", "flat_field_z%03d_sfov%03d_n2000.tif", + }; + + MultiSemFlatFieldCorrectionClient.main(args); + } + +} From 6992f596bb411c0eb190ea8a2692e36d1aa34b01 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Tue, 23 Apr 2024 17:14:34 -0400 Subject: [PATCH 07/16] Revert "Add first version of filter that does the processing" This reverts commit ef55898e2bf1017f40f9391fcfd28fd5cee28a70. The filter is not used since this should not be done dynamically. --- .../filter/ImageCalculatorFilter.java | 99 ------------------- 1 file changed, 99 deletions(-) delete mode 100644 render-app/src/main/java/org/janelia/alignment/filter/ImageCalculatorFilter.java diff --git a/render-app/src/main/java/org/janelia/alignment/filter/ImageCalculatorFilter.java b/render-app/src/main/java/org/janelia/alignment/filter/ImageCalculatorFilter.java deleted file mode 100644 index 70fb5d7bd..000000000 --- a/render-app/src/main/java/org/janelia/alignment/filter/ImageCalculatorFilter.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.janelia.alignment.filter; - -import ij.ImagePlus; -import ij.process.ImageProcessor; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.function.DoubleBinaryOperator; - -public class ImageCalculatorFilter implements Filter { - - public enum BinaryOperation { - ADD(Double::sum), - SUBTRACT((a, b) -> a - b), - MULTIPLY((a, b) -> a * b), - DIVIDE((a, b) -> a / b), - MIN(Math::min), - MAX(Math::max), - AVERAGE((a, b) -> (a + b) / 2.0), - COPY((a, b) -> b), - DIFFERENCE((a, b) -> Math.abs(a - b)); - - private final DoubleBinaryOperator operator; - - BinaryOperation(final DoubleBinaryOperator operator) { - this.operator = operator; - } - - public double apply(final double a, final double b) { - return operator.applyAsDouble(a, b); - } - - public static BinaryOperation fromString(final String operation) { - for (final BinaryOperation op : values()) { - if (op.name().equalsIgnoreCase(operation)) { - return op; - } - } - throw new IllegalArgumentException("unknown operation: " + operation); - } - } - - private String secondImageUri; - private ImageProcessor secondImage; - private BinaryOperation operation; - - // empty constructor required to create instances from specifications - @SuppressWarnings("unused") - public ImageCalculatorFilter() { - this(null, null); - } - - public ImageCalculatorFilter(final String secondImageUri, final BinaryOperation operation) { - this.secondImageUri = secondImageUri; - this.operation = operation; - initializeSecondImage(); - } - - @Override - public void init(final Map params) { - this.operation = BinaryOperation.fromString(Filter.getStringParameter("operation", params)); - this.secondImageUri = Filter.getStringParameter("secondImageUri", params); - initializeSecondImage(); - } - - private void initializeSecondImage() { - if (secondImageUri == null || operation == null) { - throw new IllegalStateException("secondImageUri and operator must be set before initializing the second image"); - } - final ImagePlus imp = new ImagePlus(secondImageUri); - secondImage = imp.getProcessor().convertToFloat(); - } - - @Override - public Map toParametersMap() { - final Map map = new LinkedHashMap<>(); - map.put("operation", operation.name()); - map.put("secondImageUri", secondImageUri); - return map; - } - - @Override - public void process(final ImageProcessor ip, final double scale) { - // TODO: check if not allowing scaling makes sense - if (scale != 1.0) { - throw new IllegalArgumentException("scaling is not supported for this filter"); - } - - // convert to 32-bit grayscale (float) for lossless processing - final ImageProcessor firstImage = ip.convertToFloat(); - - // apply the operation - for (int i = 0; i < ip.getPixelCount(); i++) { - final double a = firstImage.getf(i); - final double b = secondImage.getf(i); - ip.setf(i, (float) operation.apply(a, b)); - } - } -} From 624833b6854d5485f2ac0af4ba9eeb84c8c2a1dd Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Tue, 23 Apr 2024 17:22:39 -0400 Subject: [PATCH 08/16] Add an option to replace last few z-layer flat fields --- .../render/client/MultiSemFlatFieldCorrectionClient.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java index ac6e2a393..a98b97a7e 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java @@ -51,6 +51,8 @@ public static class Parameters extends CommandLineParameters { private String targetStackSuffix = "_corrected"; @Parameter(names = "--inputRoot", description = "Root folder for input data; if given, the structure under this root is replicated in the output folder") private String inputRoot = null; + @Parameter(names = "--flatFieldConstantFromZ", description = "Maximum z-layer of flat field estimates to consider. All subsequent z-layers get corrected with the maxium, since the estimates can be bad for the last few z-layers. If not given, all z-layers are considered") + private Integer flatFieldConstantFromZ = Integer.MAX_VALUE; } public static void main(final String[] args) { @@ -90,7 +92,7 @@ public void correctTiles() throws IOException { for (final TileSpec tileSpec : tileSpecs.getTileSpecs()) { final ImageProcessor ip = loadImageTile(tileSpec); final int sfov = extractSfovNumber(tileSpec); - final ImageProcessor flatFieldEstimate = loadFlatFieldEstimate(z, sfov); + final ImageProcessor flatFieldEstimate = loadFlatFieldEstimate(Math.min(z, params.flatFieldConstantFromZ), sfov); applyFlatFieldCorrection(ip, flatFieldEstimate); From fe39a35589a55bc87c02ca186b5f073dcf7b0bff Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Tue, 23 Apr 2024 17:30:45 -0400 Subject: [PATCH 09/16] Remove debug code --- .../render/client/MultiSemFlatFieldCorrectionClient.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java index a98b97a7e..0c1061b31 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java @@ -85,8 +85,7 @@ public void correctTiles() throws IOException { final StackMetaData stackMetaData = renderClient.getStackMetaData(stack.getStack()); renderClient.setupDerivedStack(stackMetaData, stack.getStack() + params.targetStackSuffix); -// for (int z = (int) zStatistics.getMin(); z <= (int) zStatistics.getMax(); z++) { - for (int z = 1; z <= 1; z++) { + for (int z = (int) zStatistics.getMin(); z <= (int) zStatistics.getMax(); z++) { final ResolvedTileSpecCollection tileSpecs = renderClient.getResolvedTiles(stack.getStack(), (double) z); for (final TileSpec tileSpec : tileSpecs.getTileSpecs()) { From 37b907b7185b86b368bfef7cb1633aaf5f74580a Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Wed, 24 Apr 2024 16:17:48 -0400 Subject: [PATCH 10/16] Change test parameters for client --- .../client/MultiSemFlatFieldCorrectionClientTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClientTest.java index 3d78a90a9..2fb7cb578 100644 --- a/render-ws-java-client/src/test/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClientTest.java +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClientTest.java @@ -19,11 +19,11 @@ public static void main(String[] args) { args = new String[] { "--baseDataUrl", "http://renderer-dev.int.janelia.org:8080/render-ws/v1", "--owner", "hess_wafer_53d", - "--project", "slab_000_to_009", - "--stackPattern", "s000_.*", + "--project", "slab_070_to_079", + "--stackPattern", "s070_m104", "--targetStackSuffix", "_corrected", "--inputRoot", "/nrs/hess/data/hess_wafer_53/raw/imaging/msem", - "--outputRoot", "/nrs/hess/render/flatfield/test/data", + "--outputRoot", "/nrs/hess/render/flatfield/test/corrected", "--flatFieldLocation", "/nrs/hess/render/flatfield/flatfields_n2000", "--flatFieldFormat", "flat_field_z%03d_sfov%03d_n2000.tif", }; From 60940ec1f58f022f963cb954b65f252cac1886f6 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Fri, 26 Apr 2024 11:15:15 -0400 Subject: [PATCH 11/16] Eliminate unnecessary statistic calculation --- .../MultiSemFlatFieldCorrectionClient.java | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java index 0c1061b31..785e15ded 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.nio.file.Path; -import java.util.DoubleSummaryStatistics; import java.util.List; /** @@ -81,17 +80,19 @@ public void correctTiles() throws IOException { // TODO: iterate by z-layer first to re-use the flat field estimate for all stacks? for (final StackId stack : stacks) { - final DoubleSummaryStatistics zStatistics = getZStatistics(stack); + final List zValues = renderClient.getStackZValues(stack.getStack()); + LOG.info("Considering {} with {} z-layers", stack, zValues.size()); + final StackMetaData stackMetaData = renderClient.getStackMetaData(stack.getStack()); renderClient.setupDerivedStack(stackMetaData, stack.getStack() + params.targetStackSuffix); - for (int z = (int) zStatistics.getMin(); z <= (int) zStatistics.getMax(); z++) { - final ResolvedTileSpecCollection tileSpecs = renderClient.getResolvedTiles(stack.getStack(), (double) z); + for (final double z : zValues) { + final ResolvedTileSpecCollection tileSpecs = renderClient.getResolvedTiles(stack.getStack(), z); for (final TileSpec tileSpec : tileSpecs.getTileSpecs()) { final ImageProcessor ip = loadImageTile(tileSpec); final int sfov = extractSfovNumber(tileSpec); - final ImageProcessor flatFieldEstimate = loadFlatFieldEstimate(Math.min(z, params.flatFieldConstantFromZ), sfov); + final ImageProcessor flatFieldEstimate = loadFlatFieldEstimate(Math.min(z, (double) params.flatFieldConstantFromZ), sfov); applyFlatFieldCorrection(ip, flatFieldEstimate); @@ -99,21 +100,14 @@ public void correctTiles() throws IOException { saveImage(ip, tileSpec); } - renderClient.saveResolvedTiles(tileSpecs, stack.getStack() + params.targetStackSuffix, (double) z); + renderClient.saveResolvedTiles(tileSpecs, stack.getStack() + params.targetStackSuffix, z); } renderClient.setStackState(stack.getStack() + params.targetStackSuffix, StackMetaData.StackState.COMPLETE); } } - private DoubleSummaryStatistics getZStatistics(final StackId stack) throws IOException { - final List zValues = renderClient.getStackZValues(stack.getStack()); - final DoubleSummaryStatistics zStatistics = zValues.stream().mapToDouble(Double::doubleValue).summaryStatistics(); - LOG.info("Considering {} with {} z-layers", stack, zStatistics.getCount()); - return zStatistics; - } - - private ImageProcessor loadFlatFieldEstimate(final int z, final int sfov) { - final Path imagePath = Path.of(params.flatFieldLocation, String.format(params.flatFieldFormat, z, sfov)); + private ImageProcessor loadFlatFieldEstimate(final double z, final int sfov) { + final Path imagePath = Path.of(params.flatFieldLocation, String.format(params.flatFieldFormat, (int) z, sfov)); final String imageUrl = "file:" + imagePath; final ImageLoader.LoaderType loaderType = ImageLoader.LoaderType.IMAGEJ_DEFAULT; return CACHE.get(imageUrl, 0, false, false, loaderType, null); From 049286b10c69bfe26fb0227982918d0f386593cf Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Fri, 26 Apr 2024 11:28:37 -0400 Subject: [PATCH 12/16] Introduce parameter for cache size --- .../client/MultiSemFlatFieldCorrectionClient.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java index 785e15ded..aa5a16bdf 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java @@ -31,11 +31,11 @@ */ public class MultiSemFlatFieldCorrectionClient { - // make cache large enough to hold all flat field estimates for one layer - private static final ImageProcessorCache CACHE = new ImageProcessorCache(91L * 2000L * 1748L + 1, false, false); + // make cache large enough to hold all flat field estimates for one layer private final Parameters params; private final RenderDataClient renderClient; + private final ImageProcessorCache cache; public static class Parameters extends CommandLineParameters { @ParametersDelegate @@ -52,6 +52,8 @@ public static class Parameters extends CommandLineParameters { private String inputRoot = null; @Parameter(names = "--flatFieldConstantFromZ", description = "Maximum z-layer of flat field estimates to consider. All subsequent z-layers get corrected with the maxium, since the estimates can be bad for the last few z-layers. If not given, all z-layers are considered") private Integer flatFieldConstantFromZ = Integer.MAX_VALUE; + @Parameter(names = "--cacheSizeGb", description = "Size of the image processor cache in GB (should be enough to hold one layer of flat field estimates)") + private double cacheSizeGb = 1.5; } public static void main(final String[] args) { @@ -73,6 +75,8 @@ public void runClient(final String[] args) throws IOException { public MultiSemFlatFieldCorrectionClient(final Parameters parameters) { this.params = parameters; this.renderClient = new RenderDataClient(parameters.multiProject.baseDataUrl, parameters.multiProject.owner, parameters.multiProject.project); + final double cacheSizeInBytes = 1_000_000_000 * parameters.cacheSizeGb; + this.cache = new ImageProcessorCache((long) cacheSizeInBytes, false, false); } public void correctTiles() throws IOException { @@ -110,7 +114,7 @@ private ImageProcessor loadFlatFieldEstimate(final double z, final int sfov) { final Path imagePath = Path.of(params.flatFieldLocation, String.format(params.flatFieldFormat, (int) z, sfov)); final String imageUrl = "file:" + imagePath; final ImageLoader.LoaderType loaderType = ImageLoader.LoaderType.IMAGEJ_DEFAULT; - return CACHE.get(imageUrl, 0, false, false, loaderType, null); + return cache.get(imageUrl, 0, false, false, loaderType, null); } private int extractSfovNumber(final TileSpec tileSpec) { From aa017e41597008a1667a9dd3fd8fef780a85489f Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Fri, 26 Apr 2024 12:02:04 -0400 Subject: [PATCH 13/16] Re-organize computation to make later port to spark easier --- .../MultiSemFlatFieldCorrectionClient.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java index aa5a16bdf..b4963c537 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java @@ -34,7 +34,6 @@ public class MultiSemFlatFieldCorrectionClient { // make cache large enough to hold all flat field estimates for one layer private final Parameters params; - private final RenderDataClient renderClient; private final ImageProcessorCache cache; public static class Parameters extends CommandLineParameters { @@ -74,24 +73,30 @@ public void runClient(final String[] args) throws IOException { public MultiSemFlatFieldCorrectionClient(final Parameters parameters) { this.params = parameters; - this.renderClient = new RenderDataClient(parameters.multiProject.baseDataUrl, parameters.multiProject.owner, parameters.multiProject.project); final double cacheSizeInBytes = 1_000_000_000 * parameters.cacheSizeGb; this.cache = new ImageProcessorCache((long) cacheSizeInBytes, false, false); } public void correctTiles() throws IOException { + final RenderDataClient renderClient = new RenderDataClient(params.multiProject.baseDataUrl, params.multiProject.owner, params.multiProject.project); final List stacks = params.multiProject.stackIdWithZ.getStackIdList(renderClient); // TODO: iterate by z-layer first to re-use the flat field estimate for all stacks? for (final StackId stack : stacks) { - final List zValues = renderClient.getStackZValues(stack.getStack()); - LOG.info("Considering {} with {} z-layers", stack, zValues.size()); + correctTilesForStack(params.multiProject.baseDataUrl, stack); + } + } - final StackMetaData stackMetaData = renderClient.getStackMetaData(stack.getStack()); - renderClient.setupDerivedStack(stackMetaData, stack.getStack() + params.targetStackSuffix); + public void correctTilesForStack(final String baseDataUrl, final StackId stack) throws IOException { + final RenderDataClient stackClient = new RenderDataClient(baseDataUrl, stack.getOwner(), stack.getProject()); + final List zValues = stackClient.getStackZValues(stack.getStack()); + LOG.info("Correcting tiles for {} with {} z-layers", stack, zValues.size()); + + final StackMetaData stackMetaData = stackClient.getStackMetaData(stack.getStack()); + stackClient.setupDerivedStack(stackMetaData, stack.getStack() + params.targetStackSuffix); for (final double z : zValues) { - final ResolvedTileSpecCollection tileSpecs = renderClient.getResolvedTiles(stack.getStack(), z); + final ResolvedTileSpecCollection tileSpecs = stackClient.getResolvedTiles(stack.getStack(), z); for (final TileSpec tileSpec : tileSpecs.getTileSpecs()) { final ImageProcessor ip = loadImageTile(tileSpec); @@ -104,10 +109,9 @@ public void correctTiles() throws IOException { saveImage(ip, tileSpec); } - renderClient.saveResolvedTiles(tileSpecs, stack.getStack() + params.targetStackSuffix, z); + stackClient.saveResolvedTiles(tileSpecs, stack.getStack() + params.targetStackSuffix, z); } - renderClient.setStackState(stack.getStack() + params.targetStackSuffix, StackMetaData.StackState.COMPLETE); - } + stackClient.setStackState(stack.getStack() + params.targetStackSuffix, StackMetaData.StackState.COMPLETE); } private ImageProcessor loadFlatFieldEstimate(final double z, final int sfov) { @@ -154,7 +158,6 @@ private void patchTileSpec(final TileSpec tileSpec) { newPath = Path.of(params.outputFolder).resolve(originalPath.getFileName()); } - ensureFolderExists(newPath.getParent()); final ChannelSpec firstChannel = tileSpec.getAllChannels().get(0); final ImageAndMask originalImage = firstChannel.getFirstMipmapEntry().getValue(); final ImageAndMask newImage = originalImage.copyWithImage(newPath.toString(), null, null); @@ -184,9 +187,11 @@ private ImageProcessor loadImageTile(final TileSpec tileSpec) { } private void saveImage(final ImageProcessor ip, final TileSpec tileSpec) { + final Path imagePath = Path.of(tileSpec.getImagePath()); + ensureFolderExists(imagePath.getParent()); final String tileId = tileSpec.getTileId(); final ImagePlus imp = new ImagePlus(tileId, ip); - IJ.save(imp, tileSpec.getImagePath()); + IJ.save(imp, imagePath.toString()); } private static final Logger LOG = LoggerFactory.getLogger(MultiSemFlatFieldCorrectionClient.class); From 874a3286e30e2871b8a47f77420e9abcd0eb7778 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Fri, 26 Apr 2024 12:07:29 -0400 Subject: [PATCH 14/16] Separate IO from tile spec concerns --- .../MultiSemFlatFieldCorrectionClient.java | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java index b4963c537..d57459bf7 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java @@ -16,6 +16,7 @@ import org.janelia.alignment.util.ImageProcessorCache; import org.janelia.render.client.parameter.CommandLineParameters; import org.janelia.render.client.parameter.MultiProjectParameters; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -105,7 +106,9 @@ public void correctTilesForStack(final String baseDataUrl, final StackId stack) applyFlatFieldCorrection(ip, flatFieldEstimate); - patchTileSpec(tileSpec); + final Path newPath = getNewPath(tileSpec); + ensureFolderExists(newPath.getParent()); + patchTileSpec(tileSpec, newPath); saveImage(ip, tileSpec); } @@ -147,21 +150,21 @@ private void applyFlatFieldCorrection(final ImageProcessor ip, final ImageProces ip.setPixels(0, fp); } - private void patchTileSpec(final TileSpec tileSpec) { - final Path originalPath = Path.of(tileSpec.getTileImageUrl().replaceFirst("file:", "")); - final Path newPath; + private void patchTileSpec(final TileSpec tileSpec, final Path newPath) { + final ChannelSpec firstChannel = tileSpec.getAllChannels().get(0); + final ImageAndMask originalImage = firstChannel.getFirstMipmapEntry().getValue(); + final ImageAndMask newImage = originalImage.copyWithImage(newPath.toString(), null, null); + firstChannel.putMipmap(0, newImage); + } + private Path getNewPath(final TileSpec tileSpec) { + final Path originalPath = Path.of(tileSpec.getTileImageUrl().replaceFirst("file:", "")); if (params.inputRoot != null) { final Path relativePath = Path.of(params.inputRoot).relativize(originalPath); - newPath = Path.of(params.outputFolder).resolve(relativePath); + return Path.of(params.outputFolder).resolve(relativePath); } else { - newPath = Path.of(params.outputFolder).resolve(originalPath.getFileName()); + return Path.of(params.outputFolder).resolve(originalPath.getFileName()); } - - final ChannelSpec firstChannel = tileSpec.getAllChannels().get(0); - final ImageAndMask originalImage = firstChannel.getFirstMipmapEntry().getValue(); - final ImageAndMask newImage = originalImage.copyWithImage(newPath.toString(), null, null); - firstChannel.putMipmap(0, newImage); } private void ensureFolderExists(final Path folder) { @@ -187,11 +190,9 @@ private ImageProcessor loadImageTile(final TileSpec tileSpec) { } private void saveImage(final ImageProcessor ip, final TileSpec tileSpec) { - final Path imagePath = Path.of(tileSpec.getImagePath()); - ensureFolderExists(imagePath.getParent()); final String tileId = tileSpec.getTileId(); final ImagePlus imp = new ImagePlus(tileId, ip); - IJ.save(imp, imagePath.toString()); + IJ.save(imp, tileSpec.getImagePath()); } private static final Logger LOG = LoggerFactory.getLogger(MultiSemFlatFieldCorrectionClient.class); From c1284f854b842ccf847e9e39f9e0e380e8256cb0 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Fri, 26 Apr 2024 12:14:07 -0400 Subject: [PATCH 15/16] Use existing util class to extract sfov number --- .../client/MultiSemFlatFieldCorrectionClient.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java index d57459bf7..0885564fe 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java @@ -16,7 +16,6 @@ import org.janelia.alignment.util.ImageProcessorCache; import org.janelia.render.client.parameter.CommandLineParameters; import org.janelia.render.client.parameter.MultiProjectParameters; -import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,6 +23,8 @@ import java.nio.file.Path; import java.util.List; +import org.janelia.render.client.multisem.Utilities; + /** * Apply a previously computed flat field estimate to correct shading in a Multi-SEM project. * Originally developed for the wafer_53 Multi-SEM dataset. @@ -101,7 +102,7 @@ public void correctTilesForStack(final String baseDataUrl, final StackId stack) for (final TileSpec tileSpec : tileSpecs.getTileSpecs()) { final ImageProcessor ip = loadImageTile(tileSpec); - final int sfov = extractSfovNumber(tileSpec); + final int sfov = Integer.parseInt(Utilities.getSFOVForTileId(tileSpec.getTileId())); final ImageProcessor flatFieldEstimate = loadFlatFieldEstimate(Math.min(z, (double) params.flatFieldConstantFromZ), sfov); applyFlatFieldCorrection(ip, flatFieldEstimate); @@ -124,12 +125,6 @@ private ImageProcessor loadFlatFieldEstimate(final double z, final int sfov) { return cache.get(imageUrl, 0, false, false, loaderType, null); } - private int extractSfovNumber(final TileSpec tileSpec) { - final String tileId = tileSpec.getTileId(); - final String[] parts = tileId.split("_"); - return Integer.parseInt(parts[1].substring(4)); - } - /** * Apply the flat field estimate to the input image. * @param ip the input image that is altered in place From 65c67ff1c93f5a23712f895928a898b314f78795 Mon Sep 17 00:00:00 2001 From: Michael Innerberger Date: Fri, 26 Apr 2024 16:41:09 -0400 Subject: [PATCH 16/16] Revert "Use existing util class to extract sfov number" This reverts commit c1284f854b842ccf847e9e39f9e0e380e8256cb0. The extracted substring is not only the sfov number but also includes the scan and mfov numbers --- .../client/MultiSemFlatFieldCorrectionClient.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java index 0885564fe..d57459bf7 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/MultiSemFlatFieldCorrectionClient.java @@ -16,6 +16,7 @@ import org.janelia.alignment.util.ImageProcessorCache; import org.janelia.render.client.parameter.CommandLineParameters; import org.janelia.render.client.parameter.MultiProjectParameters; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,8 +24,6 @@ import java.nio.file.Path; import java.util.List; -import org.janelia.render.client.multisem.Utilities; - /** * Apply a previously computed flat field estimate to correct shading in a Multi-SEM project. * Originally developed for the wafer_53 Multi-SEM dataset. @@ -102,7 +101,7 @@ public void correctTilesForStack(final String baseDataUrl, final StackId stack) for (final TileSpec tileSpec : tileSpecs.getTileSpecs()) { final ImageProcessor ip = loadImageTile(tileSpec); - final int sfov = Integer.parseInt(Utilities.getSFOVForTileId(tileSpec.getTileId())); + final int sfov = extractSfovNumber(tileSpec); final ImageProcessor flatFieldEstimate = loadFlatFieldEstimate(Math.min(z, (double) params.flatFieldConstantFromZ), sfov); applyFlatFieldCorrection(ip, flatFieldEstimate); @@ -125,6 +124,12 @@ private ImageProcessor loadFlatFieldEstimate(final double z, final int sfov) { return cache.get(imageUrl, 0, false, false, loaderType, null); } + private int extractSfovNumber(final TileSpec tileSpec) { + final String tileId = tileSpec.getTileId(); + final String[] parts = tileId.split("_"); + return Integer.parseInt(parts[1].substring(4)); + } + /** * Apply the flat field estimate to the input image. * @param ip the input image that is altered in place