-
Notifications
You must be signed in to change notification settings - Fork 32
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add first version of DistributedIntensityCorrectionSolver
- Loading branch information
Showing
5 changed files
with
316 additions
and
9 deletions.
There are no files selected for viewing
174 changes: 174 additions & 0 deletions
174
...c/main/java/org/janelia/render/client/newsolver/DistributedIntensityCorrectionSolver.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,174 @@ | ||
| package org.janelia.render.client.newsolver; | ||
|
|
||
| import mpicbg.models.AffineModel1D; | ||
| import mpicbg.models.TranslationModel1D; | ||
| import org.janelia.render.client.newsolver.assembly.Assembler; | ||
| import org.janelia.render.client.newsolver.assembly.ZBlockSolver; | ||
| import org.janelia.render.client.newsolver.assembly.matches.SameTileMatchCreatorAffineIntensity; | ||
| import org.janelia.render.client.newsolver.blockfactories.ZBlockFactory; | ||
| import org.janelia.render.client.newsolver.blocksolveparameters.FIBSEMIntensityCorrectionParameters; | ||
| import org.janelia.render.client.newsolver.setup.IntensityCorrectionSetup; | ||
| import org.janelia.render.client.newsolver.setup.RenderSetup; | ||
| import org.janelia.render.client.newsolver.solvers.Worker; | ||
| import org.janelia.render.client.newsolver.solvers.WorkerTools; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.concurrent.Callable; | ||
| import java.util.concurrent.ExecutionException; | ||
| import java.util.concurrent.ExecutorService; | ||
| import java.util.concurrent.Executors; | ||
|
|
||
| public class DistributedIntensityCorrectionSolver { | ||
| final IntensityCorrectionSetup cmdLineSetup; | ||
| final RenderSetup renderSetup; | ||
| BlockCollection<?, ArrayList<AffineModel1D>, ? extends FIBSEMIntensityCorrectionParameters<?>, ZBlockFactory> col; | ||
| ZBlockFactory blockFactory; | ||
|
|
||
| public DistributedIntensityCorrectionSolver( | ||
| final IntensityCorrectionSetup cmdLineSetup, | ||
| final RenderSetup renderSetup) { | ||
| this.cmdLineSetup = cmdLineSetup; | ||
| this.renderSetup = renderSetup; | ||
| } | ||
|
|
||
| public static void main(final String[] args) throws IOException { | ||
| final IntensityCorrectionSetup cmdLineSetup = new IntensityCorrectionSetup(); | ||
|
|
||
| // TODO: remove testing hack ... | ||
| if (args.length == 0) { | ||
| final String[] testArgs = { | ||
| "--baseDataUrl", "http://em-services-1.int.janelia.org:8080/render-ws/v1", | ||
| "--owner", "Z0720_07m_BR", | ||
| "--project", "Sec24", | ||
| "--stack", "v5_acquire_trimmed_align", | ||
| "--targetStack", "v5_acquire_trimmed_test", | ||
| "--completeCorrectedStack", | ||
| "--numThreads", "12", | ||
| // for entire stack minZ is 1 and maxZ is 63,300 | ||
| "--zDistance", "1", "--minZ", "1000", "--maxZ", "1001" | ||
| }; | ||
| cmdLineSetup.parse(testArgs); | ||
| } else { | ||
| cmdLineSetup.parse(args); | ||
| } | ||
|
|
||
| final RenderSetup renderSetup = RenderSetup.setupSolve(cmdLineSetup); | ||
|
|
||
| // Note: different setups can be used if specific things need to be done for the solve or certain blocks | ||
| final DistributedIntensityCorrectionSolver solverSetup = new DistributedIntensityCorrectionSolver(cmdLineSetup, renderSetup); | ||
|
|
||
| // create all block instances | ||
| final BlockCollection<?, ArrayList<AffineModel1D>, ?, ZBlockFactory> blockCollection = solverSetup.setupSolve(); | ||
|
|
||
| // | ||
| // multi-threaded solve | ||
| // | ||
| LOG.info("Multithreading with thread num=" + cmdLineSetup.threadsGlobal); | ||
|
|
||
| final ArrayList<Callable<List<BlockData<?, ArrayList<AffineModel1D>, ?, ZBlockFactory>>>> workers = new ArrayList<>(); | ||
|
|
||
|
|
||
| for (final BlockData<?, ArrayList<AffineModel1D>, ?, ZBlockFactory> block : blockCollection.allBlocks()) { | ||
| workers.add(() -> | ||
| { | ||
| final Worker<?, ArrayList<AffineModel1D>, ?, ZBlockFactory> worker = block.createWorker( | ||
| solverSetup.col.maxId() + 1, | ||
| cmdLineSetup.threadsWorker); | ||
|
|
||
| worker.run(); | ||
|
|
||
| return new ArrayList<>(worker.getBlockDataList()); | ||
| }); | ||
| } | ||
|
|
||
| final ArrayList<BlockData<?, ArrayList<AffineModel1D>, ?, ZBlockFactory>> allItems = new ArrayList<>(); | ||
|
|
||
| try { | ||
| final ExecutorService taskExecutor = Executors.newFixedThreadPool(cmdLineSetup.threadsGlobal); | ||
|
|
||
| taskExecutor.invokeAll(workers).forEach(future -> { | ||
| try { | ||
| allItems.addAll(future.get()); | ||
| } catch (final InterruptedException | ExecutionException e) { | ||
| LOG.error("Failed to compute alignments: " + e); | ||
| e.printStackTrace(); | ||
| } | ||
| }); | ||
|
|
||
| taskExecutor.shutdown(); | ||
| } catch (final InterruptedException e) { | ||
| LOG.error("Failed to compute alignments: " + e); | ||
| e.printStackTrace(); | ||
| return; | ||
| } | ||
|
|
||
| // avoid duplicate id assigned while splitting solveitems in the workers | ||
| // but do keep ids that are smaller or equal to the maxId of the initial solveset | ||
| final int maxId = WorkerTools.fixIds(allItems, solverSetup.col.maxId()); | ||
|
|
||
| LOG.info("computed " + allItems.size() + " blocks, maxId=" + maxId); | ||
|
|
||
| final ZBlockSolver<ArrayList<AffineModel1D>, TranslationModel1D, ArrayList<AffineModel1D>> solver = | ||
| new ZBlockSolver<>( | ||
| new TranslationModel1D(), | ||
| new SameTileMatchCreatorAffineIntensity(), | ||
| cmdLineSetup.distributedSolve.maxPlateauWidthGlobal, | ||
| cmdLineSetup.distributedSolve.maxAllowedErrorGlobal, | ||
| cmdLineSetup.distributedSolve.maxIterationsGlobal, | ||
| cmdLineSetup.threadsGlobal); | ||
|
|
||
| final Assembler<ArrayList<AffineModel1D>, TranslationModel1D, ArrayList<AffineModel1D>, ZBlockFactory> assembler = new Assembler<>(allItems, solver); | ||
| assembler.createAssembly(); | ||
| } | ||
|
|
||
| public <M> BlockCollection<M, ArrayList<AffineModel1D>, FIBSEMIntensityCorrectionParameters<M>, ZBlockFactory> setupSolve() { | ||
|
|
||
| final ZBlockFactory blockFactory = setupBlockFactory(); | ||
| this.blockFactory = blockFactory; | ||
|
|
||
| final BlockCollection<M, ArrayList<AffineModel1D>, FIBSEMIntensityCorrectionParameters<M>, ZBlockFactory> col = | ||
| setupBlockCollection(blockFactory); | ||
|
|
||
| this.col = col; | ||
|
|
||
| return col; | ||
| } | ||
|
|
||
| protected ZBlockFactory setupBlockFactory() { | ||
| final int minZ = (int) Math.round(renderSetup.minZ); | ||
| final int maxZ = (int) Math.round(renderSetup.maxZ); | ||
| final int blockSize = cmdLineSetup.distributedSolve.blockSize; | ||
| final int minBlockSize = cmdLineSetup.distributedSolve.minBlockSize; | ||
|
|
||
| return new ZBlockFactory(minZ, maxZ, blockSize, minBlockSize); | ||
| } | ||
|
|
||
| protected <M> FIBSEMIntensityCorrectionParameters<M> setupSolveParameters() { | ||
|
|
||
| return new FIBSEMIntensityCorrectionParameters<>( | ||
| null, | ||
| cmdLineSetup.renderWeb.baseDataUrl, | ||
| cmdLineSetup.renderWeb.owner, | ||
| cmdLineSetup.renderWeb.project, | ||
| cmdLineSetup.stack, | ||
| cmdLineSetup.intensityCorrectedFilterStack, | ||
| cmdLineSetup.maxPixelCacheGb, | ||
| cmdLineSetup.lambdaTranslation, | ||
| cmdLineSetup.lambdaIdentity, | ||
| cmdLineSetup.renderScale, | ||
| cmdLineSetup.numCoefficients, | ||
| cmdLineSetup.zDistance); | ||
| } | ||
|
|
||
| protected <M> BlockCollection<M, ArrayList<AffineModel1D>, FIBSEMIntensityCorrectionParameters<M>, ZBlockFactory> setupBlockCollection( final ZBlockFactory blockFactory){ | ||
|
|
||
| final FIBSEMIntensityCorrectionParameters<M> defaultSolveParams = setupSolveParameters(); | ||
| return blockFactory.defineBlockCollection(rtsc -> defaultSolveParams); | ||
| } | ||
|
|
||
| private static final Logger LOG = LoggerFactory.getLogger(DistributedIntensityCorrectionSolver.class); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
132 changes: 132 additions & 0 deletions
132
...ent/src/main/java/org/janelia/render/client/newsolver/setup/IntensityCorrectionSetup.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| package org.janelia.render.client.newsolver.setup; | ||
|
|
||
| import com.beust.jcommander.IStringConverter; | ||
| import com.beust.jcommander.Parameter; | ||
| import com.beust.jcommander.ParametersDelegate; | ||
| import org.janelia.render.client.intensityadjust.AdjustBlock; | ||
| import org.janelia.render.client.intensityadjust.AffineIntensityCorrectionStrategy; | ||
| import org.janelia.render.client.parameter.CommandLineParameters; | ||
| import org.janelia.render.client.parameter.RenderWebServiceParameters; | ||
| import org.janelia.render.client.parameter.ZRangeParameters; | ||
| import org.janelia.render.client.solver.SerializableValuePair; | ||
|
|
||
| import java.util.List; | ||
|
|
||
| public class IntensityCorrectionSetup extends CommandLineParameters { | ||
| private static final long serialVersionUID = -932686804562684884L; | ||
|
|
||
| @ParametersDelegate | ||
| public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); | ||
|
|
||
| @ParametersDelegate | ||
| public DistributedSolveParameters distributedSolve = new DistributedSolveParameters(); | ||
|
|
||
| @Parameter( | ||
| names = "--stack", | ||
| description = "Stack name", | ||
| required = true) | ||
| public String stack; | ||
|
|
||
| @Parameter( | ||
| names = "--intensityCorrectedFilterStack", | ||
| description = "Name of stack to store tile specs with intensity corrected filter data. " + | ||
| "Omit to render intensity corrected scape-images to disk.") | ||
| public String intensityCorrectedFilterStack; | ||
|
|
||
| @ParametersDelegate | ||
| public ZRangeParameters layerRange = new ZRangeParameters(); | ||
|
|
||
| @Parameter( | ||
| names = "--z", | ||
| description = "Explicit z values for sections to be processed", | ||
| variableArity = true) // e.g. --z 20.0 21.0 22.0 | ||
| public List<Double> zValues; | ||
|
|
||
| @Parameter( | ||
| names = "--numThreads", | ||
| description = "Number of threads to use") | ||
| public Integer numThreads = 1; | ||
|
|
||
| @Parameter( | ||
| names = "--lambdaTranslation", | ||
| description = "Lambda for regularization with translation model") | ||
| public Double lambdaTranslation = AffineIntensityCorrectionStrategy.DEFAULT_LAMBDA; | ||
|
|
||
| @Parameter( | ||
| names = "--lambdaIdentity", | ||
| description = "Lambda for regularization with identity model") | ||
| public Double lambdaIdentity = AffineIntensityCorrectionStrategy.DEFAULT_LAMBDA; | ||
|
|
||
| @Parameter( | ||
| names = { "--maxPixelCacheGb" }, | ||
| description = "Maximum number of gigabytes of pixels to cache" | ||
| ) | ||
| public Integer maxPixelCacheGb = 1; | ||
|
|
||
| @Parameter( | ||
| names = "--renderScale", | ||
| description = "Scale for rendered tiles used during intensity comparison") | ||
| public double renderScale = 0.1; | ||
|
|
||
| @Parameter( | ||
| names = "--zDistance", | ||
| description = "If specified, apply correction across this many z-layers from the current z-layer " + | ||
| "(omit to only correct in 2D)") | ||
| public Integer zDistance; | ||
|
|
||
| @Parameter( | ||
| names = { "--numCoefficients" }, | ||
| description = "Number of correction coefficients to derive in each dimension " + | ||
| "(e.g. value of 8 will divide each tile into 8x8 = 64 sub-regions)" | ||
| ) | ||
| public int numCoefficients = AdjustBlock.DEFAULT_NUM_COEFFICIENTS; | ||
|
|
||
| // | ||
| // for saving and running | ||
| // | ||
|
|
||
| @Parameter( | ||
| names = "--targetOwner", | ||
| description = "Owner name for intensity corrected result stack (default is same as owner)" | ||
| ) | ||
| public String targetOwner; | ||
|
|
||
| @Parameter( | ||
| names = "--targetProject", | ||
| description = "Project name for intensity corrected result stack (default is same as project)" | ||
| ) | ||
| public String targetProject; | ||
|
|
||
| @Parameter( | ||
| names = "--targetStack", | ||
| description = "Name for intensity corrected result stack (if omitted, models are simply logged)") | ||
| public String targetStack; | ||
|
|
||
| @Parameter( | ||
| names = "--completeTargetStack", | ||
| description = "Complete the target stack after processing", | ||
| arity = 0) | ||
| public boolean completeTargetStack = false; | ||
|
|
||
| @Parameter(names = "--threadsWorker", description = "Number of threads to be used within each worker job (default:1)") | ||
| public int threadsWorker = 1; | ||
|
|
||
| @Parameter(names = "--threadsGlobal", description = "Number of threads to be used for global intensity correction (default: numProcessors/2)") | ||
| public int threadsGlobal = Math.max( 1, Runtime.getRuntime().availableProcessors() / 2 ); | ||
|
|
||
| @Parameter( | ||
| names = "--visualizeResults", | ||
| description = "Visualize results (if running interactively)", | ||
| arity = 0) | ||
| public boolean visualizeResults = false; | ||
|
|
||
| public void initDefaultValues() { | ||
| // owner for target is the same as owner for render, if not specified otherwise | ||
| if ( this.targetOwner == null ) | ||
| this.targetOwner = renderWeb.owner; | ||
|
|
||
| // project for target is the same as project for render, if not specified otherwise | ||
| if ( this.targetProject == null ) | ||
| this.targetProject = renderWeb.project; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters