Skip to content

Commit

Permalink
fix java mfov match patch client so that it works without explicit mf…
Browse files Browse the repository at this point in the history 
…ov specification
  • Loading branch information
@trautmane
trautmane committed Jul 22, 2023
1 parent e4683fb commit a0c34ad
Show file tree
Hide file tree
Showing 5 changed files with 124 additions and 80 deletions.
47 changes: 23 additions & 24 deletions ...-client/src/main/java/org/janelia/render/client/multisem/MFOVMontageMatchPatchClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import org.janelia.alignment.spec.TileBoundsRTree;
import org.janelia.alignment.spec.TileSpec;
import org.janelia.alignment.spec.stack.StackId;
import org.janelia.alignment.spec.stack.StackWithZValues;
import org.janelia.alignment.util.FileUtil;
import org.janelia.render.client.ClientRunner;
import org.janelia.render.client.RenderDataClient;
Expand Down Expand Up @@ -80,7 +79,6 @@ public void runClient(final String[] args)

final Parameters parameters = new Parameters();
parameters.parse(args);
parameters.patch.validateAndSetupDerivedValues();

LOG.info("runClient: entry, parameters={}", parameters);

Expand All @@ -91,25 +89,26 @@ public void runClient(final String[] args)
clientRunner.run();
}

private final Parameters parameters;
private final MultiProjectParameters multiProject;
private MFOVMontageMatchPatchParameters patch;

public MFOVMontageMatchPatchClient(final Parameters parameters) {
this.parameters = parameters;
this.multiProject = parameters.multiProject;
this.patch = parameters.patch;
}

public void deriveAndSaveMatchesForAllUnconnectedPairs()
throws Exception {

final MultiProjectParameters multiProject = parameters.multiProject;
final RenderDataClient defaultDataClient = multiProject.getDataClient();
final List<StackWithZValues> stackWithZList = multiProject.buildListOfStackWithAllZ();
final List<StackMFOVWithZValues> stackMFOVWithZValuesList =
multiProject.buildListOfStackMFOVWithAllZ(patch.multiFieldOfViewId);

for (final StackWithZValues stackWithZ : stackWithZList) {
final StackMFOVWithZValues stackMFOVWithZValues =
new StackMFOVWithZValues(stackWithZ, parameters.patch.multiFieldOfViewId);
final StackId stackId = stackWithZ.getStackId();
for (final StackMFOVWithZValues stackMFOVWithZValues : stackMFOVWithZValuesList) {
final StackId stackId = stackMFOVWithZValues.getStackId();
this.patch = this.patch.withMultiFieldOfViewId(stackMFOVWithZValues.getmFOVId());
final MatchCollectionId matchCollectionId = multiProject.getMatchCollectionIdForStack(stackId);
final String matchStorageCollectionName = parameters.patch.getMatchStorageCollectionName(matchCollectionId);
final String matchStorageCollectionName = patch.getMatchStorageCollectionName(matchCollectionId);
deriveAndSaveMatchesForUnconnectedPairsInStack(defaultDataClient,
stackMFOVWithZValues,
matchCollectionId,
Expand All @@ -125,10 +124,10 @@ public void deriveAndSaveMatchesForUnconnectedPairsInStack(final RenderDataClien

LOG.info("deriveAndSaveMatchesForUnconnectedPairsInStack: entry, stackMFOVWithZValues={}", stackMFOVWithZValues);

if (! stackMFOVWithZValues.getmFOVId().equals(parameters.patch.multiFieldOfViewId)) {
if (! stackMFOVWithZValues.getmFOVId().equals(patch.multiFieldOfViewId)) {
throw new IllegalStateException("specified mFOVId " + stackMFOVWithZValues.getmFOVId() +
" differs from --multiFieldOfViewId " +
parameters.patch.multiFieldOfViewId);
patch.multiFieldOfViewId);
}

final StackId stackId = stackMFOVWithZValues.getStackId();
Expand Down Expand Up @@ -174,17 +173,17 @@ public void deriveAndSaveMatchesForUnconnectedPairsInStack(final RenderDataClien
final MFOVPositionPairMatchData positionPairMatchData = positionToPairs.get(positionPair);
derivedMatchesForMFOV.addAll(
positionPairMatchData.deriveMatchesForUnconnectedPairs(matchClient,
parameters.patch.sameLayerDerivedMatchWeight,
parameters.patch.crossLayerDerivedMatchWeight));
patch.sameLayerDerivedMatchWeight,
patch.crossLayerDerivedMatchWeight));
}

if (derivedMatchesForMFOV.size() > 0) {

LOG.info("deriveAndSaveMatchesForUnconnectedPairsInStack: saving matches for {} pairs in {}",
derivedMatchesForMFOV.size(), stackMFOVWithZValues);

if (parameters.patch.matchStorageFile != null) {
final Path storagePath = Paths.get(parameters.patch.matchStorageFile).toAbsolutePath();
if (patch.matchStorageFile != null) {
final Path storagePath = Paths.get(patch.matchStorageFile).toAbsolutePath();
FileUtil.saveJsonFile(storagePath.toString(), derivedMatchesForMFOV);
} else {
final RenderDataClient matchStorageClient = matchClient.buildClient(matchCollectionId.getOwner(),
Expand Down Expand Up @@ -218,7 +217,7 @@ public void updatePositionPairDataForZ(final String stack,

final Set<OrderedCanvasIdPair> potentialPairsForZ = tree.getCircleNeighbors(tileBoundsList,
new ArrayList<>(),
parameters.patch.xyNeighborFactor,
patch.xyNeighborFactor,
null,
false,
false,
Expand All @@ -227,8 +226,8 @@ public void updatePositionPairDataForZ(final String stack,
// add all MFOV tile pairs to unconnected set
final Set<OrderedCanvasIdPair> unconnectedPairsForMFOV = new HashSet<>(potentialPairsForZ.size());
for (final OrderedCanvasIdPair pair : potentialPairsForZ) {
if (pair.getP().getId().startsWith(parameters.patch.pTileIdPrefixForRun) &&
pair.getQ().getId().startsWith(parameters.patch.qTileIdPrefixForRun)) {
if (pair.getP().getId().startsWith(patch.pTileIdPrefixForRun) &&
pair.getQ().getId().startsWith(patch.qTileIdPrefixForRun)) {
// remove relative position info from tree search to simplify existence check later
final OrderedCanvasIdPair pairWithoutRelative =
new OrderedCanvasIdPair(pair.getP().withoutRelativePosition(),
Expand All @@ -246,7 +245,7 @@ public void updatePositionPairDataForZ(final String stack,
}

LOG.info("updatePositionPairDataForZ: found {} pairs within mFOV {} in z {}",
unconnectedPairsForMFOV.size(), parameters.patch.multiFieldOfViewId, z);
unconnectedPairsForMFOV.size(), patch.multiFieldOfViewId, z);

// query web service to find connected tile pairs and remove them from unconnected set
final Map<String, OrderedCanvasIdPair> sameLayerPairsFromOtherMFOVs = new HashMap<>();
Expand All @@ -261,8 +260,8 @@ public void updatePositionPairDataForZ(final String stack,
final OrderedCanvasIdPair pair = new OrderedCanvasIdPair(new CanvasId(groupId, pId),
new CanvasId(groupId, qId),
0.0);
if (pId.startsWith(parameters.patch.pTileIdPrefixForRun) &&
qId.startsWith(parameters.patch.qTileIdPrefixForRun)) {
if (pId.startsWith(patch.pTileIdPrefixForRun) &&
qId.startsWith(patch.qTileIdPrefixForRun)) {
if (! unconnectedPairsForMFOV.remove(pair)) {
LOG.warn("updatePositionPairDataForZ: failed to locate existing pair {} in potential set",
pair);
Expand Down Expand Up @@ -293,7 +292,7 @@ public void updatePositionPairDataForZ(final String stack,
}

LOG.info("updatePositionPairDataForZ: exit, found {} unconnected tile pairs within mFOV {} in z {}",
unconnectedPairsForMFOV.size(), parameters.patch.multiFieldOfViewId, z);
unconnectedPairsForMFOV.size(), patch.multiFieldOfViewId, z);
}

private static final Logger LOG = LoggerFactory.getLogger(MFOVMontageMatchPatchClient.class);
Expand Down
94 changes: 62 additions & 32 deletions ...nt/src/main/java/org/janelia/render/client/parameter/MFOVMontageMatchPatchParameters.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ public class MFOVMontageMatchPatchParameters

@Parameter(
names = "--mfov",
description = "Multi-field-of-view identifier <slab number>_<mfov number> (e.g. 001_000006)")
description = "Multi-field-of-view identifier <slab number>_<mfov number> (e.g. 001_000006). " +
"Omit to process all MFOVs in stack.")
public String multiFieldOfViewId;

@Parameter(
Expand Down Expand Up @@ -73,6 +74,28 @@ public class MFOVMontageMatchPatchParameters
public MFOVMontageMatchPatchParameters() {
}

/**
* @return clone of these parameters with the specified multiFieldOfViewId.
* Clone has derived values populated and validated.
*
* @throws IllegalArgumentException
* if the specified multiFieldOfViewId is inconsistent with other parameters (e.g. tileIds).
*/
public MFOVMontageMatchPatchParameters withMultiFieldOfViewId(final String multiFieldOfViewIdForClone)
throws IllegalArgumentException {
final MFOVMontageMatchPatchParameters clonedParameters = new MFOVMontageMatchPatchParameters();
clonedParameters.matchStorageCollection = this.matchStorageCollection;
clonedParameters.multiFieldOfViewId = multiFieldOfViewIdForClone;
clonedParameters.sameLayerDerivedMatchWeight = this.sameLayerDerivedMatchWeight;
clonedParameters.crossLayerDerivedMatchWeight = this.crossLayerDerivedMatchWeight;
clonedParameters.xyNeighborFactor = this.xyNeighborFactor;
clonedParameters.pTileId = this.pTileId;
clonedParameters.qTileId = this.qTileId;
clonedParameters.matchStorageFile = this.matchStorageFile;
clonedParameters.validateAndSetupDerivedValues(); // make sure values derived from multiFieldOfViewId are rebuilt
return clonedParameters;
}

// 001_000006_019_20220407_115555.1247.0 => 001_000006_019
public String getTileIdPrefixForRun(final String tileId) throws IllegalArgumentException {
if (tileId.length() < 14) {
Expand All @@ -81,39 +104,16 @@ public String getTileIdPrefixForRun(final String tileId) throws IllegalArgumentE
return tileId.substring(0, 14);
}

public void validateAndSetupDerivedValues()
throws IllegalArgumentException {

if (pTileId != null) {
multiFieldOfViewId = Utilities.getMFOVForTileId(pTileId);
pTileIdPrefixForRun = getTileIdPrefixForRun(pTileId);
if (qTileId != null) {
if (! multiFieldOfViewId.equals(Utilities.getMFOVForTileId(qTileId))) {
throw new IllegalArgumentException("pTileId and qTileId reference different MFOVs");
}
qTileIdPrefixForRun = getTileIdPrefixForRun(qTileId);
} else {
qTileIdPrefixForRun = multiFieldOfViewId;
public String getMultiFieldOfViewId() {
String mFOVId = multiFieldOfViewId;
if (mFOVId == null) {
if (pTileId != null) {
mFOVId = Utilities.getMFOVForTileId(pTileId);
} else if (qTileId != null) {
mFOVId = Utilities.getMFOVForTileId(qTileId);
}
} else if (qTileId != null) {
multiFieldOfViewId = Utilities.getMFOVForTileId(qTileId);
qTileIdPrefixForRun = getTileIdPrefixForRun(qTileId);
pTileIdPrefixForRun = multiFieldOfViewId;
} else if ((multiFieldOfViewId == null) || (multiFieldOfViewId.length() != 10)) {
throw new IllegalArgumentException("--mfov should be a 10 character value (e.g. 001_000006)");
} else {
pTileIdPrefixForRun = multiFieldOfViewId;
qTileIdPrefixForRun = multiFieldOfViewId;
}

if (matchStorageFile != null) {
Utilities.validateMatchStorageLocation(matchStorageFile);
}
}

public void setMultiFieldOfViewId(final String multiFieldOfViewId) {
this.multiFieldOfViewId = multiFieldOfViewId;
this.validateAndSetupDerivedValues(); // make sure values derived from multiFieldOfViewId are rebuilt
return mFOVId;
}

public String getMatchStorageCollectionName(final MatchCollectionId sourceCollectionId) {
Expand Down Expand Up @@ -141,6 +141,36 @@ public static MFOVMontageMatchPatchParameters fromJsonFile(final String dataFile
return parameters;
}

private void validateAndSetupDerivedValues()
throws IllegalArgumentException {

if (pTileId != null) {
multiFieldOfViewId = Utilities.getMFOVForTileId(pTileId);
pTileIdPrefixForRun = getTileIdPrefixForRun(pTileId);
if (qTileId != null) {
if (! multiFieldOfViewId.equals(Utilities.getMFOVForTileId(qTileId))) {
throw new IllegalArgumentException("pTileId and qTileId reference different MFOVs");
}
qTileIdPrefixForRun = getTileIdPrefixForRun(qTileId);
} else {
qTileIdPrefixForRun = multiFieldOfViewId;
}
} else if (qTileId != null) {
multiFieldOfViewId = Utilities.getMFOVForTileId(qTileId);
qTileIdPrefixForRun = getTileIdPrefixForRun(qTileId);
pTileIdPrefixForRun = multiFieldOfViewId;
} else if ((multiFieldOfViewId == null) || (multiFieldOfViewId.length() != 10)) {
throw new IllegalArgumentException("--mfov should be a 10 character value (e.g. 001_000006)");
} else {
pTileIdPrefixForRun = multiFieldOfViewId;
qTileIdPrefixForRun = multiFieldOfViewId;
}

if (matchStorageFile != null) {
Utilities.validateMatchStorageLocation(matchStorageFile);
}
}

private static final JsonUtils.Helper<MFOVMontageMatchPatchParameters> JSON_HELPER =
new JsonUtils.Helper<>(MFOVMontageMatchPatchParameters.class);
}
34 changes: 34 additions & 0 deletions ...java-client/src/main/java/org/janelia/render/client/parameter/MultiProjectParameters.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,17 @@

import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;

import org.janelia.alignment.match.MatchCollectionId;
import org.janelia.alignment.multisem.StackMFOVWithZValues;
import org.janelia.alignment.spec.stack.StackId;
import org.janelia.alignment.spec.stack.StackWithZValues;
import org.janelia.render.client.RenderDataClient;
import org.janelia.render.client.multisem.Utilities;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* Parameters for identifying projects, stacks, and match collections.
Expand Down Expand Up @@ -90,10 +95,39 @@ public List<StackWithZValues> buildListOfStackWithAllZ()
return stackIdWithZ.buildListOfStackWithAllZ(this.getDataClient());
}

public List<StackMFOVWithZValues> buildListOfStackMFOVWithAllZ(final String multiFieldOfViewId)
throws IOException, IllegalArgumentException {

final List<StackMFOVWithZValues> stackMFOVWithZValuesList = new ArrayList<>();

final RenderDataClient defaultRenderClient = getDataClient();
final List<StackWithZValues> stackWithZValuesList = buildListOfStackWithAllZ();
for (final StackWithZValues stackWithZValues : stackWithZValuesList) {
final StackId stackId = stackWithZValues.getStackId();
final RenderDataClient dataClient = defaultRenderClient.buildClient(stackId.getOwner(),
stackId.getProject());
final List<String> mFOVIdList = Utilities.getMFOVNames(dataClient,
stackId.getStack(),
stackWithZValues.getFirstZ());

LOG.info("buildListOfStackMFOVWithAllZ: found {} MFOVs in {}", mFOVIdList.size(), stackWithZValues);

for (final String mFOVId : mFOVIdList) {
if ((multiFieldOfViewId == null) || multiFieldOfViewId.equals(mFOVId)) {
stackMFOVWithZValuesList.add(new StackMFOVWithZValues(stackWithZValues, mFOVId));
}
}
}

return stackMFOVWithZValuesList;
}

private synchronized void buildDataClient() {
if (dataClient == null) {
dataClient = new RenderDataClient(baseDataUrl, owner, project);
}
}

private static final Logger LOG = LoggerFactory.getLogger(MultiProjectParameters.class);

}
4 changes: 2 additions & 2 deletions ...ent/src/test/java/org/janelia/render/client/multisem/MFOVMontageMatchPatchClientTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ public static void main(final String[] args) {
"--xyNeighborFactor", "0.6", // for tiles completely within mFov
"--sameLayerDerivedMatchWeight", "0.15",
"--crossLayerDerivedMatchWeight", "0.1",
"--pTileId", "145_000010_001_20220618_040953.80.0",
"--qTileId", "145_000010_002_20220618_040953.80.0",
// "--pTileId", "145_000010_001_20220618_040953.80.0",
// "--qTileId", "145_000010_002_20220618_040953.80.0",

"--matchCollection", "c001_s145_v01_match",
"--matchStorageCollection", "c001_s145_v01_match_try3",
Expand Down
Loading

0 comments on commit a0c34ad

Please sign in to comment.