From 4f745f92e3c2ad73976ce15a268eb1774bae5b75 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Fri, 31 Jul 2026 13:29:29 -0400 Subject: [PATCH 01/40] add convertTileIdToUseWafers6061RenderOrder method --- .../alignment/multisem/MultiSemUtilities.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/render-app/src/main/java/org/janelia/alignment/multisem/MultiSemUtilities.java b/render-app/src/main/java/org/janelia/alignment/multisem/MultiSemUtilities.java index 7bca85591..f752f525e 100644 --- a/render-app/src/main/java/org/janelia/alignment/multisem/MultiSemUtilities.java +++ b/render-app/src/main/java/org/janelia/alignment/multisem/MultiSemUtilities.java @@ -263,6 +263,36 @@ public static boolean isSimpleMFOVName(final String name) { /** Each MFOV has 91 SFOVs or tiles */ public static int NUMBER_OF_TILES_IN_MFOV = 91; + /** + * @return the specified tileId with its render order updated + * (e.g.'w60_magc0399_scan005_m0013_r42_s09' for 'w60_magc0399_scan005_m0013_r37_s09') + */ + public static String convertTileIdToUseWafers6061RenderOrder(final String tileId) { + final String sfovString = getSFOVIndexForTileId(tileId); // "09" for "w60_magc0399_scan005_m0013_r37_s09" + final int i = Integer.parseInt(sfovString.trim()) - 1; // 8 for "09" + final String renderOrderString = WAFERS_60_61_RENDER_ORDER_STRINGS[i]; // "r42_" for 8 + return tileId.substring(0, 27) + renderOrderString + sfovString; + } + + /** + * Thomas' optimal SFOV render order for wafers 60 and 61 that was copied from + * + * render_sfov_order.py + * . + */ + private static final String[] WAFERS_60_61_RENDER_ORDER_STRINGS = { + "r57_s", "r50_s", "r46_s", "r51_s", "r60_s", "r65_s", "r59_s", "r47_s", "r42_s", "r36_s", + "r43_s", "r48_s", "r54_s", "r56_s", "r69_s", "r76_s", "r72_s", "r68_s", "r55_s", "r44_s", + "r37_s", "r31_s", "r24_s", "r32_s", "r38_s", "r18_s", "r25_s", "r33_s", "r39_s", "r64_s", + "r71_s", "r83_s", "r79_s", "r75_s", "r70_s", "r63_s", "r52_s", "r34_s", "r26_s", "r19_s", + "r12_s", "r07_s", "r20_s", "r27_s", "r13_s", "r08_s", "r14_s", "r21_s", "r28_s", "r35_s", + "r45_s", "r67_s", "r78_s", "r86_s", "r88_s", "r85_s", "r82_s", "r77_s", "r66_s", "r58_s", + "r49_s", "r29_s", "r22_s", "r15_s", "r09_s", "r03_s", "r01_s", "r04_s", "r16_s", "r10_s", + "r05_s", "r02_s", "r06_s", "r11_s", "r17_s", "r23_s", "r30_s", "r40_s", "r62_s", "r74_s", + "r81_s", "r90_s", "r91_s", "r89_s", "r87_s", "r84_s", "r80_s", "r73_s", "r61_s", "r53_s", + "r41_s" + }; + private static final Pattern SIMPLE_MFOV_NAME_PATTERN = Pattern.compile("^m(\\d{4})$"); private static final Logger LOG = LoggerFactory.getLogger(MultiSemUtilities.class); From 70239e1ac70838f2e92d0bbc6fb3c5fc6abf83f6 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Fri, 31 Jul 2026 14:07:42 -0400 Subject: [PATCH 02/40] move MaskHackClient from mask package to tile package --- .../render/client/spark/pipeline/AlignmentPipelineStepId.java | 2 +- .../render/client/spark/{mask => tile}/MaskHackClient.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename render-ws-spark-client/src/main/java/org/janelia/render/client/spark/{mask => tile}/MaskHackClient.java (99%) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java index 6a5878d34..b408ae9c2 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java @@ -4,7 +4,7 @@ import org.janelia.render.client.spark.MipmapClient; import org.janelia.render.client.spark.ScapeClient; -import org.janelia.render.client.spark.mask.MaskHackClient; +import org.janelia.render.client.spark.tile.MaskHackClient; import org.janelia.render.client.spark.match.ClusterCountClient; import org.janelia.render.client.spark.match.CopyMatchClient; import org.janelia.render.client.spark.match.MultiStagePointMatchClient; diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/mask/MaskHackClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/MaskHackClient.java similarity index 99% rename from render-ws-spark-client/src/main/java/org/janelia/render/client/spark/mask/MaskHackClient.java rename to render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/MaskHackClient.java index 8d2df52a4..11a2fdb6d 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/mask/MaskHackClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/MaskHackClient.java @@ -1,4 +1,4 @@ -package org.janelia.render.client.spark.mask; +package org.janelia.render.client.spark.tile; import com.beust.jcommander.ParametersDelegate; From e8ccb4d0577e0f8614979d38b150ab6e6522e227 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Mon, 3 Aug 2026 10:57:30 -0400 Subject: [PATCH 03/40] add TileIdHackClient --- .../pipeline/AlignmentPipelineStepId.java | 2 + .../client/spark/tile/TileIdHackClient.java | 163 ++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java index b408ae9c2..aa886c495 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java @@ -16,6 +16,7 @@ import org.janelia.render.client.spark.newsolver.DistributedAffineBlockSolverClient; import org.janelia.render.client.spark.newsolver.DistributedIntensityCorrectionBlockSolverClient; import org.janelia.render.client.spark.tile.RenderTilesClient; +import org.janelia.render.client.spark.tile.TileIdHackClient; import org.janelia.render.client.spark.zspacing.ZPositionCorrectionClient; /** @@ -36,6 +37,7 @@ public enum AlignmentPipelineStepId { CORRECT_Z_POSITIONS(ZPositionCorrectionClient::new), CORRECT_INTENSITY(DistributedIntensityCorrectionBlockSolverClient::new), HACK_MASK(MaskHackClient::new), + HACK_TILE_ID(TileIdHackClient::new), RENDER_SCAPE_IMAGES(ScapeClient::new), RENDER_TILES(RenderTilesClient::new), MFOV_AS_TILE(MFOVASTileClient::new), diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java new file mode 100644 index 000000000..d66f00319 --- /dev/null +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java @@ -0,0 +1,163 @@ +package org.janelia.render.client.spark.tile; + +import com.beust.jcommander.ParametersDelegate; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.Function; +import org.janelia.alignment.multisem.MultiSemUtilities; +import org.janelia.alignment.spec.ResolvedTileSpecCollection; +import org.janelia.alignment.spec.TileSpec; +import org.janelia.alignment.spec.stack.StackId; +import org.janelia.alignment.spec.stack.StackIdNamingGroup; +import org.janelia.alignment.spec.stack.StackMetaData; +import org.janelia.alignment.spec.stack.StackWithZValues; +import org.janelia.render.client.ClientRunner; +import org.janelia.render.client.RenderDataClient; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MultiProjectParameters; +import org.janelia.render.client.spark.LogUtilities; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineParameters; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineStep; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineStepId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Spark client for updating the tileId for all tiles in the stack + * to use the new wafer 60 and 61 render order. + */ +public class TileIdHackClient + implements Serializable, AlignmentPipelineStep { + + public static class Parameters extends CommandLineParameters { + @ParametersDelegate + public MultiProjectParameters multiProject; + + public Parameters() { + this(new MultiProjectParameters()); + } + + public Parameters(final MultiProjectParameters multiProject) { + this.multiProject = multiProject; + } + } + + /** Run the client with command line parameters. */ + public static void main(final String[] args) { + final ClientRunner clientRunner = new ClientRunner(args) { + @Override + public void runClient(final String[] args) throws Exception { + final Parameters parameters = new Parameters(); + parameters.parse(args); + final TileIdHackClient client = new TileIdHackClient(); + client.createContextAndRun(parameters); + } + }; + clientRunner.run(); + } + + /** Empty constructor required for alignment pipeline steps. */ + public TileIdHackClient() { + } + + /** Create a spark context and run the client with the specified parameters. */ + public void createContextAndRun(final Parameters clientParameters) throws IOException { + final SparkConf conf = new SparkConf().setAppName(getClass().getSimpleName()); + try (final JavaSparkContext sparkContext = new JavaSparkContext(conf)) { + LOG.info("createContextAndRun: appId is {}", sparkContext.getConf().getAppId()); + copyStackAndFixTileIds(sparkContext, clientParameters); + } + } + + /** Validates the specified pipeline parameters are sufficient. */ + @Override + public void validatePipelineParameters(final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException { + // nothing to validate + } + + /** Run the client as part of an alignment pipeline. */ + public void runPipelineStep(final JavaSparkContext sparkContext, + final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException, IOException { + + final StackIdNamingGroup rawNamingGroup = pipelineParameters.getRawNamingGroup(); + if (rawNamingGroup == null) { + throw new IllegalArgumentException( + "The " + AlignmentPipelineStepId.HACK_TILE_ID + " pipeline step requires that " + + "a raw pipelineStackGroup is defined in the pipeline parameters."); + } + final MultiProjectParameters multiProject = pipelineParameters.getMultiProject(rawNamingGroup); + final Parameters clientParameters = new Parameters(multiProject); + copyStackAndFixTileIds(sparkContext, clientParameters); + } + + @Override + public AlignmentPipelineStepId getDefaultStepId() { + return AlignmentPipelineStepId.HACK_TILE_ID; + } + + private void copyStackAndFixTileIds(final JavaSparkContext sparkContext, + final Parameters clientParameters) + throws IOException { + + LOG.info("copyStackAndFixTileIds: entry, clientParameters={}", clientParameters); + + final MultiProjectParameters multiProjectParameters = clientParameters.multiProject; + final String baseDataUrl = multiProjectParameters.getBaseDataUrl(); + final List stackWithZValuesList = multiProjectParameters.buildListOfStackWithAllZ(); + + final JavaRDD rddStackWithZValues = sparkContext.parallelize(stackWithZValuesList); + + final Function copyAndFixTileIdFunction = stackWithZ -> { + + LogUtilities.setupExecutorLog4j(stackWithZ.toString()); + + final StackId stackId = stackWithZ.getStackId(); + final String stack = stackId.getStack(); + final RenderDataClient sourceDataClient = new RenderDataClient(baseDataUrl, + stackId.getOwner(), + stackId.getProject()); + + sourceDataClient.setStackState(stack, StackMetaData.StackState.LOADING); + + for (final Double z : stackWithZ.getzValues()) { + final ResolvedTileSpecCollection resolvedTiles = sourceDataClient.getResolvedTiles(stack, z); + + // tile specs are keyed by tileId within a collection, so build a new collection after fixing the ids + final List tileSpecsWithFixedIds = new ArrayList<>(resolvedTiles.getTileSpecs()); + tileSpecsWithFixedIds.forEach(tileSpec -> { + final String fixedTileId = + MultiSemUtilities.convertTileIdToUseWafers6061RenderOrder(tileSpec.getTileId()); + tileSpec.setTileId(fixedTileId); + }); + + final ResolvedTileSpecCollection resolvedTilesWithFixedIds = + new ResolvedTileSpecCollection(resolvedTiles.getTransformSpecs(), + tileSpecsWithFixedIds); + + sourceDataClient.deleteStack(stack, z); // remove original tile specs for z layer + sourceDataClient.saveResolvedTiles(resolvedTilesWithFixedIds, stack, z); // save new tile specs for z layer + } + + sourceDataClient.setStackState(stack, StackMetaData.StackState.COMPLETE); + + return null; + }; + + final JavaRDD rddCopyAndFixTileId = rddStackWithZValues.map(copyAndFixTileIdFunction); + rddCopyAndFixTileId.collect(); + + LOG.info("copyStackAndFixTileIds: collected rddCopyAndFixTileId"); + LOG.info("copyStackAndFixTileIds: exit"); + } + + private static final Logger LOG = LoggerFactory.getLogger(TileIdHackClient.class); +} From c2f30334c3ca2d401c1d7e55378deef8eb2c517d Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Mon, 3 Aug 2026 15:43:27 -0400 Subject: [PATCH 04/40] add excludeDifferentMfovNeighbors parameter for match tile pair derivation --- .../parameters/TilePairDerivationParameters.java | 6 ++++++ .../janelia/alignment/spec/TileBoundsRTree.java | 14 ++++++++++---- .../alignment/spec/TileBoundsRTreeTest.java | 6 ++++++ .../client/DebugTransformedCornersClient.java | 1 + .../client/ExampleMatchVisualizationClient.java | 2 ++ .../janelia/render/client/RestartSolverClient.java | 2 ++ .../org/janelia/render/client/TilePairClient.java | 3 ++- .../MFOVAsTileMontageMatchPatchClient.java | 1 + .../multisem/MFOVMontageMatchPatchClient.java | 1 + 9 files changed, 31 insertions(+), 5 deletions(-) diff --git a/render-app/src/main/java/org/janelia/alignment/match/parameters/TilePairDerivationParameters.java b/render-app/src/main/java/org/janelia/alignment/match/parameters/TilePairDerivationParameters.java index 6fa7df579..e4db5f302 100644 --- a/render-app/src/main/java/org/janelia/alignment/match/parameters/TilePairDerivationParameters.java +++ b/render-app/src/main/java/org/janelia/alignment/match/parameters/TilePairDerivationParameters.java @@ -73,6 +73,12 @@ public class TilePairDerivationParameters implements Serializable { arity = 1) public boolean excludeSameMfovNeighbors = false; + @Parameter( + names = "--excludeDifferentMfovNeighbors", + description = "Exclude neighbor tiles in the same layer (z) with a different MFOV from the source tile", + arity = 1) + public boolean excludeDifferentMfovNeighbors = false; + @Parameter( names = "--excludePairsInMatchCollection", description = "Name of match collection whose existing pairs should be excluded from the generated list (default is to include all pairs)" diff --git a/render-app/src/main/java/org/janelia/alignment/spec/TileBoundsRTree.java b/render-app/src/main/java/org/janelia/alignment/spec/TileBoundsRTree.java index 34be18ed6..cbee6d0fd 100644 --- a/render-app/src/main/java/org/janelia/alignment/spec/TileBoundsRTree.java +++ b/render-app/src/main/java/org/janelia/alignment/spec/TileBoundsRTree.java @@ -219,7 +219,8 @@ public Set getCircleNeighbors(final List source final boolean excludeCornerNeighbors, final boolean excludeSameLayerNeighbors, final boolean excludeSameSectionNeighbors, - final boolean excludeSameMfovNeighbors) { + final boolean excludeSameMfovNeighbors, + final boolean excludeDifferentMfovNeighbors) { String firstTileId = null; if (! sourceTileBoundsList.isEmpty()) { @@ -258,7 +259,8 @@ public Set getCircleNeighbors(final List source neighborTileIdPairs.addAll( getDistinctPairs(tileBounds, searchResults, - excludeCornerNeighbors, excludeSameSectionNeighbors, excludeSameMfovNeighbors, + excludeCornerNeighbors, excludeSameSectionNeighbors, + excludeSameMfovNeighbors, excludeDifferentMfovNeighbors, true)); } @@ -266,7 +268,8 @@ public Set getCircleNeighbors(final List source searchResults = neighborTree.findTilesInCircle(circle); neighborTileIdPairs.addAll( getDistinctPairs(tileBounds, searchResults, - excludeCornerNeighbors, excludeSameSectionNeighbors, excludeSameMfovNeighbors, + excludeCornerNeighbors, excludeSameSectionNeighbors, + excludeSameMfovNeighbors, excludeDifferentMfovNeighbors, false)); } } @@ -310,10 +313,11 @@ public static Set getDistinctPairs(final TileBounds fromTil final boolean excludeCornerNeighbors, final boolean excludeSameSectionNeighbors, final boolean excludeSameMfovNeighbors, + final boolean excludeDifferentMfovNeighbors, final boolean includeRelativePosition) { final Set pairs = new HashSet<>(toTiles.size() * 2); final String pTileId = fromTile.getTileId(); - final String pMfov = excludeSameMfovNeighbors ? MultiSemUtilities.getMagcMfovForTileId(pTileId) : null; + final String pMfov = (excludeSameMfovNeighbors || excludeDifferentMfovNeighbors) ? MultiSemUtilities.getMagcMfovForTileId(pTileId) : null; final double fromMinX = fromTile.getMinX(); final double fromMaxX = fromTile.getMaxX(); @@ -334,6 +338,8 @@ public static Set getDistinctPairs(final TileBounds fromTil if (excludeSameMfovNeighbors && (pMfov.equals(MultiSemUtilities.getMagcMfovForTileId(qTileId)))) { continue; + } else if (excludeDifferentMfovNeighbors && (! pMfov.equals(MultiSemUtilities.getMagcMfovForTileId(qTileId)))) { + continue; } final OrderedCanvasIdPair pair; diff --git a/render-app/src/test/java/org/janelia/alignment/spec/TileBoundsRTreeTest.java b/render-app/src/test/java/org/janelia/alignment/spec/TileBoundsRTreeTest.java index 9db430fcf..e3316f051 100644 --- a/render-app/src/test/java/org/janelia/alignment/spec/TileBoundsRTreeTest.java +++ b/render-app/src/test/java/org/janelia/alignment/spec/TileBoundsRTreeTest.java @@ -117,6 +117,7 @@ public void testGetCircleNeighbors() { false, false, false, + false, false); final Set expectedPairs = new TreeSet<>(); @@ -138,6 +139,7 @@ public void testGetCircleNeighbors() { false, false, false, + false, false); expectedPairs.clear(); @@ -168,6 +170,7 @@ public void testGetCircleNeighborsWithFullyOverlappingTiles() { false, false, false, + false, false); final Set expectedPairs = new TreeSet<>(); @@ -197,6 +200,7 @@ public void testGetCanvasIdPairs() { false, false, false, + false, false); int expectedNumberOfCombinations = tileBoundsList.size() - 1; // all tiles except the center Assert.assertEquals("incorrect number of combinations (with corner neighbors) in " + pairs, @@ -208,6 +212,7 @@ public void testGetCanvasIdPairs() { true, false, false, + false, true); Assert.assertEquals("incorrect number of combinations (without corner neighbors) in " + pairs, expectedNumberOfCombinations, pairs.size()); @@ -236,6 +241,7 @@ public void testGetDifferentMFovCanvasIdPairs() { false, false, true, + false, false); Assert.assertEquals("incorrect number of combinations in " + pairs, 2, pairs.size()); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/DebugTransformedCornersClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/DebugTransformedCornersClient.java index fdade1bfe..e576e9842 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/DebugTransformedCornersClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/DebugTransformedCornersClient.java @@ -263,6 +263,7 @@ private List debugPairsForStack(final String stackName) false, false, false, + false, false); neighborPairs.addAll(currentNeighborPairs); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/ExampleMatchVisualizationClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/ExampleMatchVisualizationClient.java index 3f0138b2f..8e556bfe5 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/ExampleMatchVisualizationClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/ExampleMatchVisualizationClient.java @@ -209,6 +209,7 @@ void printConnections() true, false, false, + false, false) .stream() .map(pair -> { @@ -246,6 +247,7 @@ void printConnections() true, true, false, + false, false) .stream().sorted().collect(Collectors.toList()); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/RestartSolverClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/RestartSolverClient.java index b2db6a8d8..645ae4ed9 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/RestartSolverClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/RestartSolverClient.java @@ -333,6 +333,7 @@ private Set getSameLayerPairs(final Map getCrossLayerPairs(final Map sorte tpdp.excludeCornerNeighbors, tpdp.excludeSameLayerNeighbors, tpdp.excludeSameSectionNeighbors, - tpdp.excludeSameMfovNeighbors); + tpdp.excludeSameMfovNeighbors, + tpdp.excludeDifferentMfovNeighbors); if (tpdp.excludeSameLayerPairsWithPosition != null) { final MontageRelativePosition excludedPosition = tpdp.excludeSameLayerPairsWithPosition; diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVAsTileMontageMatchPatchClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVAsTileMontageMatchPatchClient.java index 300a0680f..a363e2d5b 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVAsTileMontageMatchPatchClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVAsTileMontageMatchPatchClient.java @@ -153,6 +153,7 @@ private static int patchUnconnectedPairsForZ(final String stack, false, false, false, + false, false); // add all tile pairs to unconnected set to start diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVMontageMatchPatchClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVMontageMatchPatchClient.java index 4972782bc..b4426bfdd 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVMontageMatchPatchClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVMontageMatchPatchClient.java @@ -258,6 +258,7 @@ public void updatePositionPairDataForZ(final String stack, false, false, false, + false, false); // add all MFOV tile pairs to unconnected set to start From c3115a38f510d625a02eebe3c4958ad045037d5c Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Mon, 3 Aug 2026 17:30:26 -0400 Subject: [PATCH 05/40] move most ThomasCalibrationIntensityCorrectionClient parameters into separate BeamCorrectionParameters class that can be more easily reused by spark pipelines --- ...sCalibrationIntensityCorrectionClient.java | 200 +++++++----------- .../parameter/BeamCorrectionParameters.java | 94 ++++++++ 2 files changed, 169 insertions(+), 125 deletions(-) create mode 100644 render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java index b1231b0b2..9d10c1e6a 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java @@ -17,13 +17,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import net.imglib2.IterableInterval; -import net.imglib2.RandomAccess; -import net.imglib2.RandomAccessibleInterval; -import net.imglib2.img.basictypeaccess.AccessFlags; -import net.imglib2.type.numeric.RealType; -import net.imglib2.type.numeric.real.FloatType; - import org.janelia.alignment.filter.FilterSpec; import org.janelia.alignment.filter.LinearIntensityMap8BitFilter; import org.janelia.alignment.spec.ResolvedTileSpecCollection; @@ -31,9 +24,9 @@ import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.render.client.ClientRunner; import org.janelia.render.client.RenderDataClient; +import org.janelia.render.client.parameter.BeamCorrectionParameters; import org.janelia.render.client.parameter.CommandLineParameters; import org.janelia.render.client.parameter.RenderWebServiceParameters; -import org.janelia.render.client.parameter.ZRangeParameters; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.N5Reader; @@ -44,6 +37,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import net.imglib2.IterableInterval; +import net.imglib2.RandomAccess; +import net.imglib2.RandomAccessibleInterval; +import net.imglib2.img.basictypeaccess.AccessFlags; +import net.imglib2.type.numeric.RealType; +import net.imglib2.type.numeric.real.FloatType; + /** * Java client that applies the pre-computed degree-0 (spatially flat) beam-homogenization correction * for a Multi-SEM stack. @@ -82,71 +82,11 @@ public static class Parameters extends CommandLineParameters { @ParametersDelegate public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - @ParametersDelegate - public ZRangeParameters layerRange = new ZRangeParameters(); - @Parameter(names = "--stack", description = "Source stack to correct", required = true) public String stack; - @Parameter(names = "--targetStack", description = "Name of the derived stack to store the corrected " + - "tile specs (defaults to _ic)") - public String targetStack; - - @Parameter(names = "--zarrPath", description = "Path to the multi-SEM acquisition zarr (xlog) container " + - "holding the intensity correction parameters and the scan/slab/sfov coordinate arrays " + - "(e.g. /path/to/xlog_wafer_61.zarr)", required = true) - public String zarrPath; - - @Parameter(names = "--homogenizationDataset", description = "Name of the 4D correction-parameter array " + - "within the zarr container") - public String homogenizationDataset = "beam_homogenization"; - - @Parameter(names = "--scanDataset", description = "Name of the 1D scan coordinate array") - public String scanDataset = "scan"; - - @Parameter(names = "--slabDataset", description = "Name of the 1D slab coordinate array " + - "(labeled by the magc number of each tile)") - public String slabDataset = "slab"; - - @Parameter(names = "--sfovDataset", description = "Name of the 1D sfov coordinate array") - public String sfovDataset = "sfov"; - - @Parameter(names = "--serialDataset", description = "Name of the 1D serial-section coordinate array " + - "(only used to cross-check / log the slab selection)") - public String serialDataset = "id_serial"; - - @Parameter(names = "--gainIndex", description = "Index of the gain parameter within the " + - "homogenization_parameter dimension") - public int gainIndex = 21; - - @Parameter(names = "--deg0Index", description = "Index of the degree-0 flat-level parameter within the " + - "homogenization_parameter dimension") - public int deg0Index = 22; - - @Parameter(names = "--referenceLevel", description = "Reference intensity level to map to " + - "(defaults to the 'b_ref' attribute of the homogenization array)") - public Double referenceLevel; - - @Parameter(names = "--inverted", description = "Indicates that the source images are intensity-inverted " + - "(in' = 255 - in) relative to the data the homogenization parameters were computed for. When set, " + - "the correction is applied in the original (non-inverted) domain and the result is re-inverted, so " + - "the slope stays gain but the offset becomes 255*(1 - gain) - (referenceLevel - gain*degree0).", arity = 0) - public boolean inverted = false; - - @Parameter(names = "--sfovLabelOffset", description = "Offset added to the sFOV number parsed from the " + - "tile id to obtain the xlog sfov coordinate label. Render tile ids are 1-based (s01..s91) while " + - "the xlog sfov coordinate is 0-based (0..90), so the default is -1.") - public int sfovLabelOffset = -1; - - @Parameter(names = "--z", description = "Explicit z values for sections to be processed", variableArity = true) - public List zValues; - - @Parameter(names = "--completeStack", description = "Complete the target stack after processing", arity = 0) - public boolean completeStack = false; - - public String getTargetStack() { - return (targetStack == null || targetStack.isEmpty()) ? stack + "_ic" : targetStack; - } + @ParametersDelegate + public BeamCorrectionParameters beam = new BeamCorrectionParameters(); } public static void main(final String[] args) { @@ -157,8 +97,14 @@ public void runClient(final String[] args) throws Exception { parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); - final ThomasCalibrationIntensityCorrectionClient client = new ThomasCalibrationIntensityCorrectionClient(parameters); - client.correctStack(); + parameters.beam.validate(); + + final ThomasCalibrationIntensityCorrectionClient client = new ThomasCalibrationIntensityCorrectionClient(); + final RenderDataClient dataClient = parameters.renderWeb.getDataClient(); + client.correctStack(dataClient, + parameters.stack, + parameters.beam, + true); } }; clientRunner.run(); @@ -172,68 +118,65 @@ public void runClient(final String[] args) throws Exception { private static final double IMAGE_MAX = 255.0; private static final String B_REF_KEY = "b_ref"; - private final Parameters params; - - public ThomasCalibrationIntensityCorrectionClient(final Parameters params) { - this.params = params; + public ThomasCalibrationIntensityCorrectionClient() { } - public void correctStack() throws IOException { + public void correctStack(final RenderDataClient dataClient, + final String stack, + final BeamCorrectionParameters beam, + final boolean completeStack) throws IOException { + - final RenderDataClient dataClient = params.renderWeb.getDataClient(); - final String targetStack = params.getTargetStack(); + final String targetStack = stack + beam.targetStackSuffix; - try (final N5Reader reader = new N5Factory().openReader(params.zarrPath)) { - LOG.info("correctStack: opened {} using {}", params.zarrPath, reader.getClass().getSimpleName()); + try (final N5Reader reader = new N5Factory().openReader(beam.zarrPath)) { + LOG.info("correctStack: opened {} using {}", beam.zarrPath, reader.getClass().getSimpleName()); - final double referenceLevel = resolveReferenceLevel(); + final double referenceLevel = resolveReferenceLevel(beam); // coordinate-value (label) -> array-position lookups - final Map scanLabelToPosition = readCoordinateIndex(reader, params.scanDataset); - final Map slabLabelToPosition = readCoordinateIndex(reader, params.slabDataset); - final Map sfovLabelToPosition = readCoordinateIndex(reader, params.sfovDataset); - final int[] serialBySlabPosition = readOptionalIntArray(reader, params.serialDataset); + final Map scanLabelToPosition = readCoordinateIndex(reader, beam.scanDataset, beam); + final Map slabLabelToPosition = readCoordinateIndex(reader, beam.slabDataset, beam); + final Map sfovLabelToPosition = readCoordinateIndex(reader, beam.sfovDataset, beam); + final int[] serialBySlabPosition = readOptionalIntArray(reader, beam.serialDataset, beam); LOG.info("correctStack: reference level (b_ref) is {}, source data inverted is {}; coordinate arrays loaded - {} scans, {} slabs, {} sfovs", - referenceLevel, params.inverted, scanLabelToPosition.size(), slabLabelToPosition.size(), sfovLabelToPosition.size()); + referenceLevel, beam.inverted, scanLabelToPosition.size(), slabLabelToPosition.size(), sfovLabelToPosition.size()); // the 4D correction array; axes are matched to coordinate sizes so the code is robust to axis order - final RandomAccessibleInterval> homogenization = openHomogenizationArray(reader, params.homogenizationDataset); + final RandomAccessibleInterval> homogenization = openHomogenizationArray(reader, beam.homogenizationDataset); final int scanAxis = findAxisForSize(homogenization, scanLabelToPosition.size(), "scan"); final int slabAxis = findAxisForSize(homogenization, slabLabelToPosition.size(), "slab"); final int sfovAxis = findAxisForSize(homogenization, sfovLabelToPosition.size(), "sfov"); final int parameterAxis = remainingAxis(homogenization, scanAxis, slabAxis, sfovAxis); final long parameterCount = homogenization.dimension(parameterAxis); - if (params.gainIndex < 0 || params.gainIndex >= parameterCount - || params.deg0Index < 0 || params.deg0Index >= parameterCount) { - throw new IllegalArgumentException("gainIndex " + params.gainIndex + " and deg0Index " + params.deg0Index + + if (beam.gainIndex < 0 || beam.gainIndex >= parameterCount + || beam.deg0Index < 0 || beam.deg0Index >= parameterCount) { + throw new IllegalArgumentException("gainIndex " + beam.gainIndex + " and deg0Index " + beam.deg0Index + " must both be within the parameter dimension of size " + parameterCount); } LOG.info("correctStack: '{}' axis mapping is scan={}, slab={}, sfov={}, parameter={} (size {}); using gainIndex={}, deg0Index={}", - params.homogenizationDataset, scanAxis, slabAxis, sfovAxis, parameterAxis, parameterCount, - params.gainIndex, params.deg0Index); + beam.homogenizationDataset, scanAxis, slabAxis, sfovAxis, parameterAxis, parameterCount, + beam.gainIndex, beam.deg0Index); final RandomAccess> access = homogenization.randomAccess(); final long[] position = new long[homogenization.numDimensions()]; - final List zValues = dataClient.getStackZValues(params.stack, - params.layerRange.minZ, - params.layerRange.maxZ, - params.zValues); + final List zValues = dataClient.getStackZValues(stack); if (zValues.isEmpty()) { - throw new IllegalArgumentException("source stack " + params.stack + " does not contain any matching z values"); + throw new IllegalArgumentException("source stack " + stack + " does not contain any matching z values"); } - final StackMetaData sourceStackMetaData = dataClient.getStackMetaData(params.stack); + final StackMetaData sourceStackMetaData = dataClient.getStackMetaData(stack); dataClient.setupDerivedStack(sourceStackMetaData, targetStack); int correctedCount = 0; int skippedCount = 0; for (final Double z : zValues) { - final ResolvedTileSpecCollection resolvedTiles = dataClient.getResolvedTiles(params.stack, z); + final ResolvedTileSpecCollection resolvedTiles = dataClient.getResolvedTiles(stack, z); for (final TileSpec tileSpec : resolvedTiles.getTileSpecs()) { final String tileId = tileSpec.getTileId(); @@ -241,7 +184,7 @@ public void correctStack() throws IOException { final int magc = parseValue(MAGC_PATTERN, tileId, "magc"); final int scan = parseValue(SCAN_PATTERN, tileId, "scan"); final int sfov = parseValue(SFOV_PATTERN, tileId, "sfov"); - final int sfovLabel = sfov + params.sfovLabelOffset; + final int sfovLabel = sfov + beam.sfovLabelOffset; final Integer scanPosition = scanLabelToPosition.get(scan); final Integer slabPosition = slabLabelToPosition.get(magc); @@ -258,11 +201,11 @@ public void correctStack() throws IOException { position[slabAxis] = slabPosition; position[sfovAxis] = sfovPosition; - position[parameterAxis] = params.gainIndex; + position[parameterAxis] = beam.gainIndex; access.setPosition(position); final double gain = access.get().getRealDouble(); - position[parameterAxis] = params.deg0Index; + position[parameterAxis] = beam.deg0Index; access.setPosition(position); final double degree0 = access.get().getRealDouble(); @@ -281,7 +224,7 @@ public void correctStack() throws IOException { tileId, scanPosition, slabPosition, serialInfo, sfovPosition, gain, degree0); } - tileSpec.setFilterSpec(buildHomogenizationFilterSpec(gain, degree0, referenceLevel, params.inverted)); + tileSpec.setFilterSpec(buildHomogenizationFilterSpec(gain, degree0, referenceLevel, beam.inverted)); tileSpec.convertSingleChannelSpecToLegacyForm(); correctedCount++; } @@ -290,12 +233,12 @@ public void correctStack() throws IOException { LOG.info("correctStack: saved z {} to {}", z, targetStack); } - if (params.completeStack) { + if (completeStack) { dataClient.setStackState(targetStack, StackMetaData.StackState.COMPLETE); } LOG.info("correctStack: exit, applied degree-0 homogenization to {} tiles and skipped {} tiles across {} layers of {} (target stack {})", - correctedCount, skippedCount, zValues.size(), params.stack, targetStack); + correctedCount, skippedCount, zValues.size(), stack, targetStack); } } @@ -324,29 +267,29 @@ private static FilterSpec buildHomogenizationFilterSpec(final double gain, return FilterSpec.forFilter(filter); } - private double resolveReferenceLevel() { - if (params.referenceLevel != null) { - return params.referenceLevel; + private double resolveReferenceLevel(final BeamCorrectionParameters beam) { + if (beam.referenceLevel != null) { + return beam.referenceLevel; } // NOTE: n5-zarr's reader.getAttribute(...) throws an NPE on blosc-compressed arrays because it // serializes the compressor, so read the b_ref value directly from the on-disk JSON metadata. - Double bRef = readDoubleFromJsonFile(Paths.get(params.zarrPath, params.homogenizationDataset, ".zattrs"), + Double bRef = readDoubleFromJsonFile(Paths.get(beam.zarrPath, beam.homogenizationDataset, ".zattrs"), B_REF_KEY); if (bRef == null) { - bRef = readBRefFromConsolidatedMetadata(); + bRef = readBRefFromConsolidatedMetadata(beam); } if (bRef == null) { throw new IllegalArgumentException("could not read '" + B_REF_KEY + "' from the " + - params.homogenizationDataset + " metadata under " + params.zarrPath + + beam.homogenizationDataset + " metadata under " + beam.zarrPath + "; specify --referenceLevel explicitly"); } LOG.info("resolveReferenceLevel: using reference level {}={} from zarr metadata", B_REF_KEY, bRef); return bRef; } - private Double readBRefFromConsolidatedMetadata() { + private Double readBRefFromConsolidatedMetadata(final BeamCorrectionParameters beam) { try { - final Path zMetadata = Paths.get(params.zarrPath, ".zmetadata"); + final Path zMetadata = Paths.get(beam.zarrPath, ".zmetadata"); if (!Files.isRegularFile(zMetadata)) { return null; } @@ -355,7 +298,7 @@ private Double readBRefFromConsolidatedMetadata() { if (metadata == null) { return null; } - final JsonObject attrs = metadata.getAsJsonObject(params.homogenizationDataset + "/.zattrs"); + final JsonObject attrs = metadata.getAsJsonObject(beam.homogenizationDataset + "/.zattrs"); return (attrs != null && attrs.has(B_REF_KEY)) ? attrs.get(B_REF_KEY).getAsDouble() : null; } catch (final Exception e) { return null; @@ -383,8 +326,10 @@ private static int parseValue(final Pattern pattern, final String tileId, final } /** Reads a 1D coordinate array and returns a map from each stored label value to its array position. */ - private Map readCoordinateIndex(final N5Reader reader, final String dataset) { - final double[] values = readCoordinateValues(reader, dataset); + private static Map readCoordinateIndex(final N5Reader reader, + final String dataset, + final BeamCorrectionParameters beam) { + final double[] values = readCoordinateValues(reader, dataset, beam); if (values == null) { throw new IllegalArgumentException("coordinate array '" + dataset + "' not found in the zarr container; " + "point --zarrPath at a container that physically contains the scan/slab/sfov coordinate arrays"); @@ -397,10 +342,12 @@ private Map readCoordinateIndex(final N5Reader reader, final S } /** Reads a 1D array into an int array indexed by position, or returns null if the dataset is absent. */ - private int[] readOptionalIntArray(final N5Reader reader, final String dataset) { + private static int[] readOptionalIntArray(final N5Reader reader, + final String dataset, + final BeamCorrectionParameters beam) { final double[] values; try { - values = readCoordinateValues(reader, dataset); + values = readCoordinateValues(reader, dataset, beam); } catch (final Exception e) { LOG.warn("readOptionalIntArray: could not read {} ({}), slab cross-check will be unavailable", dataset, e.getMessage()); return null; @@ -425,8 +372,10 @@ private int[] readOptionalIntArray(final N5Reader reader, final String dataset) * that are physically present), the resulting {@link ZarrDatasetAttributes} is built via the reader, and the * chunks are read with {@link N5Reader#readBlock} (which honors the zarr little-endian byte order). */ - private double[] readCoordinateValues(final N5Reader reader, final String dataset) { - final JsonObject zArray = readZArrayJson(dataset); + private static double[] readCoordinateValues(final N5Reader reader, + final String dataset, + final BeamCorrectionParameters beam) { + final JsonObject zArray = readZArrayJson(dataset, beam); if (zArray == null) { return null; } @@ -451,7 +400,7 @@ private double[] readCoordinateValues(final N5Reader reader, final String datase final double[] values = new double[length]; final int numChunks = (int) Math.ceil((double) length / chunkSize); for (int chunk = 0; chunk < numChunks; chunk++) { - final DataBlock block = reader.readBlock(dataset, attributes, (long) chunk); + final DataBlock block = reader.readBlock(dataset, attributes, chunk); if (block == null) { throw new IllegalArgumentException("missing chunk " + chunk + " of coordinate array " + dataset); } @@ -461,8 +410,9 @@ private double[] readCoordinateValues(final N5Reader reader, final String datase } /** Reads the raw .zarray JSON for a dataset from the per-array file, falling back to consolidated .zmetadata. */ - private JsonObject readZArrayJson(final String dataset) { - final Path perArray = Paths.get(params.zarrPath, dataset, ".zarray"); + private static JsonObject readZArrayJson(final String dataset, + final BeamCorrectionParameters beam) { + final Path perArray = Paths.get(beam.zarrPath, dataset, ".zarray"); try { if (Files.isRegularFile(perArray)) { return JsonParser.parseString(Files.readString(perArray)).getAsJsonObject(); @@ -471,7 +421,7 @@ private JsonObject readZArrayJson(final String dataset) { LOG.warn("readZArrayJson: failed to read {} ({})", perArray, e.getMessage()); } try { - final Path zMetadata = Paths.get(params.zarrPath, ".zmetadata"); + final Path zMetadata = Paths.get(beam.zarrPath, ".zmetadata"); if (Files.isRegularFile(zMetadata)) { final JsonObject root = JsonParser.parseString(Files.readString(zMetadata)).getAsJsonObject(); final JsonObject metadata = root.getAsJsonObject("metadata"); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java new file mode 100644 index 000000000..f872e399b --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java @@ -0,0 +1,94 @@ +package org.janelia.render.client.parameter; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import java.io.Serializable; + +/** + * Parameters for correcting multi-SEM image creep. + */ +@Parameters +public class BeamCorrectionParameters + implements Serializable { + + @Parameter( + names = "--zarrPath", + description = "Path to the multi-SEM acquisition zarr (xlog) container " + + "holding the intensity correction parameters and the scan/slab/sfov coordinate arrays " + + "(e.g. /path/to/xlog_wafer_61.zarr)", required = true) + public String zarrPath; + + @Parameter( + names = "--homogenizationDataset", + description = "Name of the 4D correction-parameter array within the zarr container") + public String homogenizationDataset = "beam_homogenization"; + + @Parameter( + names = "--scanDataset", + description = "Name of the 1D scan coordinate array") + public String scanDataset = "scan"; + + @Parameter( + names = "--slabDataset", + description = "Name of the 1D slab coordinate array (labeled by the magc number of each tile)") + public String slabDataset = "slab"; + + @Parameter( + names = "--sfovDataset", + description = "Name of the 1D sfov coordinate array") + public String sfovDataset = "sfov"; + + @Parameter( + names = "--serialDataset", + description = "Name of the 1D serial-section coordinate array (only used to cross-check / log the slab selection)") + public String serialDataset = "id_serial"; + + @Parameter( + names = "--gainIndex", + description = "Index of the gain parameter within the homogenization_parameter dimension") + public int gainIndex = 21; + + @Parameter( + names = "--deg0Index", + description = "Index of the degree-0 flat-level parameter within the homogenization_parameter dimension") + public int deg0Index = 22; + + @Parameter( + names = "--referenceLevel", + description = "Reference intensity level to map to (defaults to the 'b_ref' attribute of the homogenization array)") + public Double referenceLevel; + + @Parameter( + names = "--inverted", + description = "Indicates that the source images are intensity-inverted " + + "(in' = 255 - in) relative to the data the homogenization parameters were computed for. When set, " + + "the correction is applied in the original (non-inverted) domain and the result is re-inverted, so " + + "the slope stays gain but the offset becomes 255*(1 - gain) - (referenceLevel - gain*degree0).", + arity = 0) + public boolean inverted = false; + + @Parameter( + names = "--sfovLabelOffset", + description = "Offset added to the sFOV number parsed from the " + + "tile id to obtain the xlog sfov coordinate label. Render tile ids are 1-based (s01..s91) while " + + "the xlog sfov coordinate is 0-based (0..90), so the default is -1.") + public int sfovLabelOffset = -1; + + @Parameter( + names = "--targetStackSuffix", + description = "Target stack name is the the source stack name with this suffix appended") + public String targetStackSuffix = "_bc"; + + public BeamCorrectionParameters() { + } + + public void validate() + throws IllegalArgumentException { + + if ((targetStackSuffix == null) || (targetStackSuffix.trim().isEmpty())) { + throw new IllegalArgumentException("--targetStackSuffix must be defined"); + } + + } +} \ No newline at end of file From 7c2ca70c0e659b5a4f0f2c117b578905504806e6 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Mon, 3 Aug 2026 19:13:15 -0400 Subject: [PATCH 06/40] add toString method to CreepCorrectionParameters --- .../client/parameter/CreepCorrectionParameters.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java index 6198b14c5..afc19f31f 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java @@ -60,6 +60,16 @@ public void validate() } } + @Override + public String toString() { + return "{targetStackSuffix='" + targetStackSuffix + '\'' + + ", minZ=" + minZ + + ", maxZ=" + maxZ + + ", skipMatchCorrection=" + skipMatchCorrection + + ", parameterCsvDir='" + parameterCsvDir + '\'' + + '}'; + } + public String getTargetStack(final String sourceStack) { return sourceStack + targetStackSuffix; } From 8b0a2ffa15f5900defff1949de71eb88759cb8cd Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Mon, 3 Aug 2026 19:14:03 -0400 Subject: [PATCH 07/40] add toString method to BeamCorrectionParameters --- .../parameter/BeamCorrectionParameters.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java index f872e399b..d07bd0eff 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java @@ -91,4 +91,21 @@ public void validate() } } + + @Override + public String toString() { + return "{zarrPath='" + zarrPath + '\'' + + ", homogenizationDataset='" + homogenizationDataset + '\'' + + ", scanDataset='" + scanDataset + '\'' + + ", slabDataset='" + slabDataset + '\'' + + ", sfovDataset='" + sfovDataset + '\'' + + ", serialDataset='" + serialDataset + '\'' + + ", gainIndex=" + gainIndex + + ", deg0Index=" + deg0Index + + ", referenceLevel=" + referenceLevel + + ", inverted=" + inverted + + ", sfovLabelOffset=" + sfovLabelOffset + + ", targetStackSuffix='" + targetStackSuffix + '\'' + + '}'; + } } \ No newline at end of file From bbcd2393abd1953f9c05e719bd9d803712f142e1 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Mon, 3 Aug 2026 20:03:46 -0400 Subject: [PATCH 08/40] first draft of BeamCorrectionSparkClient --- .../multisem/BeamCorrectionSparkClient.java | 144 ++++++++++++++++++ .../pipeline/AlignmentPipelineParameters.java | 9 ++ .../pipeline/AlignmentPipelineStepId.java | 4 +- 3 files changed, 155 insertions(+), 2 deletions(-) create mode 100644 render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java new file mode 100644 index 000000000..2e5eac43f --- /dev/null +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java @@ -0,0 +1,144 @@ +package org.janelia.render.client.spark.multisem; + +import com.beust.jcommander.ParametersDelegate; + +import java.io.IOException; +import java.io.Serializable; +import java.util.List; + +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.janelia.alignment.spec.stack.StackId; +import org.janelia.render.client.ClientRunner; +import org.janelia.render.client.RenderDataClient; +import org.janelia.render.client.multisem.ThomasCalibrationIntensityCorrectionClient; +import org.janelia.render.client.parameter.BeamCorrectionParameters; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MultiProjectParameters; +import org.janelia.render.client.spark.LogUtilities; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineParameters; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineStep; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineStepId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Spark client for applying the pre-computed degree-0 beam-homogenization correction to multi-SEM stacks. + * + * @see ThomasCalibrationIntensityCorrectionClient + */ +public class BeamCorrectionSparkClient + implements Serializable, AlignmentPipelineStep { + + public static class Parameters extends CommandLineParameters { + + @ParametersDelegate + public MultiProjectParameters multiProject = new MultiProjectParameters(); + + @ParametersDelegate + public BeamCorrectionParameters beamCorrection = new BeamCorrectionParameters(); + } + + public static void main(final String[] args) { + final ClientRunner clientRunner = new ClientRunner(args) { + @Override + public void runClient(final String[] args) throws Exception { + final Parameters parameters = new Parameters(); + parameters.parse(args); + parameters.beamCorrection.validate(); + + LOG.info("runClient: entry, parameters={}", parameters); + + final BeamCorrectionSparkClient client = new BeamCorrectionSparkClient(); + client.createContextAndRun(parameters); + } + }; + clientRunner.run(); + } + + public BeamCorrectionSparkClient() { + } + + /** + * Create a spark context and run the client with the specified parameters. + */ + public void createContextAndRun(final Parameters clientParameters) + throws IOException { + final SparkConf conf = new SparkConf().setAppName(getClass().getSimpleName()); + try (final JavaSparkContext sparkContext = new JavaSparkContext(conf)) { + + LOG.info("createContextAndRun: appId is {}", sparkContext.getConf().getAppId()); + + correctBeamIntensity(sparkContext, + clientParameters.multiProject, + clientParameters.beamCorrection); + } + } + + /** Validates the specified pipeline parameters are sufficient. */ + @Override + public void validatePipelineParameters(final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException { + final BeamCorrectionParameters beamCorrection = pipelineParameters.getBeamCorrection(); + AlignmentPipelineParameters.validateRequiredElementExists("beamCorrection", + beamCorrection); + beamCorrection.validate(); + } + + /** Run the client as part of an alignment pipeline. */ + @Override + public void runPipelineStep(final JavaSparkContext sparkContext, + final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException, IOException { + + final MultiProjectParameters multiProject = + pipelineParameters.getMultiProject(pipelineParameters.getRawNamingGroup()); + + correctBeamIntensity(sparkContext, + multiProject, + pipelineParameters.getBeamCorrection()); + } + + @Override + public AlignmentPipelineStepId getDefaultStepId() { + return AlignmentPipelineStepId.CORRECT_BEAM_INTENSITY; + } + + public void correctBeamIntensity(final JavaSparkContext sparkContext, + final MultiProjectParameters multiProject, + final BeamCorrectionParameters beamCorrection) throws IOException { + + final String baseDataUrl = multiProject.getBaseDataUrl(); + + // all z values of each stack are corrected, so only the stack ids are needed here + final List stackIdList = multiProject.stackIdWithZ.getStackIdList(multiProject.getDataClient()); + + LOG.info("correctBeamIntensity: entry, distributing {} stacks with beamCorrection {}", + stackIdList.size(), beamCorrection); + + if (stackIdList.isEmpty()) { + throw new IllegalArgumentException("no stacks match the specified parameters"); + } + + final JavaRDD rddStackIds = sparkContext.parallelize(stackIdList); + + rddStackIds.foreach(stackId -> { + + LogUtilities.setupExecutorLog4j(stackId.toDevString()); + + final RenderDataClient executorDataClient = new RenderDataClient(baseDataUrl, + stackId.getOwner(), + stackId.getProject()); + + new ThomasCalibrationIntensityCorrectionClient().correctStack(executorDataClient, + stackId.getStack(), + beamCorrection, + true); + }); + + LOG.info("correctBeamIntensity: exit, corrected {} stacks", stackIdList.size()); + } + + private static final Logger LOG = LoggerFactory.getLogger(BeamCorrectionSparkClient.class); +} diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java index e19edddd1..1aa8f6dd7 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java @@ -16,6 +16,7 @@ import org.janelia.alignment.util.UrlResourceUtil; import org.janelia.render.client.newsolver.setup.AffineBlockSolverSetup; import org.janelia.render.client.newsolver.setup.IntensityCorrectionSetup; +import org.janelia.render.client.parameter.BeamCorrectionParameters; import org.janelia.render.client.parameter.CreepCorrectionParameters; import org.janelia.render.client.parameter.LayerAsTileParameters; import org.janelia.render.client.parameter.MFOVAsTileParameters; @@ -50,6 +51,7 @@ public class AlignmentPipelineParameters private final UnconnectedCrossMFOVParameters unconnectedCrossMfov; private final TileClusterParameters tileCluster; private final MatchCopyParameters matchCopy; + private final BeamCorrectionParameters beamCorrection; private final CreepCorrectionParameters creepCorrection; private final AffineBlockSolverSetup affineBlockSolverSetup; private final IntensityCorrectionSetup intensityCorrectionSetup; @@ -79,6 +81,7 @@ public AlignmentPipelineParameters() { null, null, null, + null, null); } @@ -91,6 +94,7 @@ public AlignmentPipelineParameters(final MultiProjectParameters multiProject, final UnconnectedCrossMFOVParameters unconnectedCrossMfov, final TileClusterParameters tileCluster, final MatchCopyParameters matchCopy, + final BeamCorrectionParameters beamCorrection, final CreepCorrectionParameters creepCorrection, final AffineBlockSolverSetup affineBlockSolverSetup, final IntensityCorrectionSetup intensityCorrectionSetup, @@ -109,6 +113,7 @@ public AlignmentPipelineParameters(final MultiProjectParameters multiProject, this.unconnectedCrossMfov = unconnectedCrossMfov; this.tileCluster = tileCluster; this.matchCopy = matchCopy; + this.beamCorrection = beamCorrection; this.creepCorrection = creepCorrection; this.affineBlockSolverSetup = affineBlockSolverSetup; this.intensityCorrectionSetup = intensityCorrectionSetup; @@ -166,6 +171,10 @@ public MatchCopyParameters getMatchCopy() { return matchCopy; } + public BeamCorrectionParameters getBeamCorrection() { + return beamCorrection; + } + public CreepCorrectionParameters getCreepCorrection() { return creepCorrection; } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java index aa886c495..ed354cbc4 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java @@ -8,6 +8,7 @@ import org.janelia.render.client.spark.match.ClusterCountClient; import org.janelia.render.client.spark.match.CopyMatchClient; import org.janelia.render.client.spark.match.MultiStagePointMatchClient; +import org.janelia.render.client.spark.multisem.BeamCorrectionSparkClient; import org.janelia.render.client.spark.multisem.CreepCorrectionSparkClient; import org.janelia.render.client.spark.multisem.LayerAsTileClient; import org.janelia.render.client.spark.multisem.MFOVASTileClient; @@ -21,8 +22,6 @@ /** * Identifier for a step in a spark alignment pipeline with a convenience {@link #toStepClient()} builder. - * - * @author Eric Trautman */ public enum AlignmentPipelineStepId { @@ -32,6 +31,7 @@ public enum AlignmentPipelineStepId { FIND_UNCONNECTED_CROSS_MFOVS(UnconnectedCrossMFOVClient::new), FIND_UNCONNECTED_TILES_AND_EDGES(ClusterCountClient::new), FILTER_MATCHES(CopyMatchClient::new), + CORRECT_BEAM_INTENSITY(BeamCorrectionSparkClient::new), CORRECT_CREEP(CreepCorrectionSparkClient::new), ALIGN_TILES(DistributedAffineBlockSolverClient::new), CORRECT_Z_POSITIONS(ZPositionCorrectionClient::new), From 05387946c2376e9b7535ab6bffba5b2118f2b1d8 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Tue, 4 Aug 2026 12:45:14 -0400 Subject: [PATCH 09/40] modify ThomasCalibrationIntensityCorrectionClient to support reading xlog zarr data from Google cloud storage --- ...sCalibrationIntensityCorrectionClient.java | 161 +++++++++--------- .../parameter/BeamCorrectionParameters.java | 6 +- 2 files changed, 83 insertions(+), 84 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java index 9d10c1e6a..901d33097 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java @@ -2,14 +2,11 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.ParametersDelegate; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import com.google.gson.JsonParser; import com.google.gson.JsonPrimitive; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -30,8 +27,10 @@ import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5URI; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.janelia.saalfeldlab.n5.universe.N5Factory; +import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; import org.janelia.saalfeldlab.n5.zarr.ZarrDatasetAttributes; import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueReader; import org.slf4j.Logger; @@ -49,7 +48,9 @@ * for a Multi-SEM stack. *

* The correction parameters are read from a multi-SEM acquisition zarr container (the "xlog", - * e.g. xlog_wafer_61.zarr). The {@code beam_homogenization} array has dimensions + * e.g. xlog_wafer_61.zarr) that is opened with {@link N5Factory} and can therefore be either a local + * file system path or a cloud URI (e.g. gs://janelia-spark-test/xlog_data/wafer_61.zarr). + * The {@code beam_homogenization} array has dimensions * {@code [scan, slab, sfov, homogenization_parameter]}; the parameter dimension (length 26) is laid out * (per the data author's code, JaneliaSciComp/EM_recon_pipeline PR #186) as: *

    @@ -117,6 +118,7 @@ public void runClient(final String[] args) throws Exception { private static final double IMAGE_MAX = 255.0; private static final String B_REF_KEY = "b_ref"; + private static final String ZMETADATA_FILE = ".zmetadata"; public ThomasCalibrationIntensityCorrectionClient() { } @@ -129,16 +131,18 @@ public void correctStack(final RenderDataClient dataClient, final String targetStack = stack + beam.targetStackSuffix; - try (final N5Reader reader = new N5Factory().openReader(beam.zarrPath)) { + try (final N5Reader reader = new N5Factory().openReader(StorageFormat.ZARR, beam.zarrPath)) { LOG.info("correctStack: opened {} using {}", beam.zarrPath, reader.getClass().getSimpleName()); - final double referenceLevel = resolveReferenceLevel(beam); + final ZarrKeyValueReader zarrReader = asZarrReader(reader, beam.zarrPath); + + final double referenceLevel = resolveReferenceLevel(zarrReader, beam); // coordinate-value (label) -> array-position lookups - final Map scanLabelToPosition = readCoordinateIndex(reader, beam.scanDataset, beam); - final Map slabLabelToPosition = readCoordinateIndex(reader, beam.slabDataset, beam); - final Map sfovLabelToPosition = readCoordinateIndex(reader, beam.sfovDataset, beam); - final int[] serialBySlabPosition = readOptionalIntArray(reader, beam.serialDataset, beam); + final Map scanLabelToPosition = readCoordinateIndex(zarrReader, beam.scanDataset); + final Map slabLabelToPosition = readCoordinateIndex(zarrReader, beam.slabDataset); + final Map sfovLabelToPosition = readCoordinateIndex(zarrReader, beam.sfovDataset); + final int[] serialBySlabPosition = readOptionalIntArray(zarrReader, beam.serialDataset); LOG.info("correctStack: reference level (b_ref) is {}, source data inverted is {}; coordinate arrays loaded - {} scans, {} slabs, {} sfovs", referenceLevel, beam.inverted, scanLabelToPosition.size(), slabLabelToPosition.size(), sfovLabelToPosition.size()); @@ -267,16 +271,18 @@ private static FilterSpec buildHomogenizationFilterSpec(final double gain, return FilterSpec.forFilter(filter); } - private double resolveReferenceLevel(final BeamCorrectionParameters beam) { + private double resolveReferenceLevel(final ZarrKeyValueReader reader, + final BeamCorrectionParameters beam) { if (beam.referenceLevel != null) { return beam.referenceLevel; } // NOTE: n5-zarr's reader.getAttribute(...) throws an NPE on blosc-compressed arrays because it - // serializes the compressor, so read the b_ref value directly from the on-disk JSON metadata. - Double bRef = readDoubleFromJsonFile(Paths.get(beam.zarrPath, beam.homogenizationDataset, ".zattrs"), - B_REF_KEY); + // serializes the compressor, so read the b_ref value directly from the JSON metadata. + Double bRef = readDouble(readJsonResource(reader, beam.homogenizationDataset, ZarrKeyValueReader.ZATTRS_FILE), + B_REF_KEY); if (bRef == null) { - bRef = readBRefFromConsolidatedMetadata(beam); + bRef = readDouble(readConsolidatedJson(reader, beam.homogenizationDataset, ZarrKeyValueReader.ZATTRS_FILE), + B_REF_KEY); } if (bRef == null) { throw new IllegalArgumentException("could not read '" + B_REF_KEY + "' from the " + @@ -287,34 +293,60 @@ private double resolveReferenceLevel(final BeamCorrectionParameters beam) { return bRef; } - private Double readBRefFromConsolidatedMetadata(final BeamCorrectionParameters beam) { + /** + * Reads a JSON metadata resource (e.g. {@code .zarray} or {@code .zattrs}) for a dataset, + * falling back to the consolidated {@code .zmetadata} at the container root. + *

    + * The resource is read with the container's {@code KeyValueAccess} instead of {@link java.nio.file.Files} + * so that metadata can be read from a local file system path or from a cloud URI + * (e.g. {@code gs://bucket/xlog_wafer_61.zarr}). + * + * @return the parsed resource or null if it is not present in the container. + */ + private static JsonObject readDatasetJson(final ZarrKeyValueReader reader, + final String dataset, + final String resourceName) { + + final JsonObject json = readJsonResource(reader, dataset, resourceName); + return json == null ? readConsolidatedJson(reader, dataset, resourceName) : json; + } + + /** Reads a dataset's JSON metadata resource from the consolidated {@code .zmetadata} at the container root. */ + private static JsonObject readConsolidatedJson(final ZarrKeyValueReader reader, + final String dataset, + final String resourceName) { + final JsonObject consolidated = readJsonResource(reader, "", ZMETADATA_FILE); + final JsonObject metadata = consolidated == null ? null : consolidated.getAsJsonObject("metadata"); + final JsonObject datasetJson = metadata == null ? null : metadata.getAsJsonObject(dataset + "/" + resourceName); + return datasetJson == null ? null : datasetJson.deepCopy(); + } + + private static Double readDouble(final JsonObject json, + final String key) { + return ((json != null) && json.has(key)) ? json.get(key).getAsDouble() : null; + } + + /** Reads one JSON resource from the container, or returns null if it is missing or cannot be parsed. */ + private static JsonObject readJsonResource(final ZarrKeyValueReader reader, + final String parentPath, + final String resourceName) { try { - final Path zMetadata = Paths.get(beam.zarrPath, ".zmetadata"); - if (!Files.isRegularFile(zMetadata)) { - return null; - } - final JsonObject root = JsonParser.parseString(Files.readString(zMetadata)).getAsJsonObject(); - final JsonObject metadata = root.getAsJsonObject("metadata"); - if (metadata == null) { - return null; - } - final JsonObject attrs = metadata.getAsJsonObject(beam.homogenizationDataset + "/.zattrs"); - return (attrs != null && attrs.has(B_REF_KEY)) ? attrs.get(B_REF_KEY).getAsDouble() : null; + final JsonElement element = reader.getAttributesFromContainer(N5URI.normalizeGroupPath(parentPath), + resourceName); + return ((element != null) && element.isJsonObject()) ? element.getAsJsonObject() : null; } catch (final Exception e) { + LOG.warn("readJsonResource: failed to read {} for '{}' ({})", resourceName, parentPath, e.getMessage()); return null; } } - private static Double readDoubleFromJsonFile(final Path path, final String key) { - try { - if (!Files.isRegularFile(path)) { - return null; - } - final JsonObject obj = JsonParser.parseString(Files.readString(path)).getAsJsonObject(); - return obj.has(key) ? obj.get(key).getAsDouble() : null; - } catch (final Exception e) { - return null; + private static ZarrKeyValueReader asZarrReader(final N5Reader reader, + final String zarrPath) { + if (!(reader instanceof ZarrKeyValueReader)) { + throw new IllegalArgumentException("expected a zarr reader for " + zarrPath + + " but got " + reader.getClass().getName()); } + return (ZarrKeyValueReader) reader; } private static int parseValue(final Pattern pattern, final String tileId, final String label) { @@ -326,10 +358,9 @@ private static int parseValue(final Pattern pattern, final String tileId, final } /** Reads a 1D coordinate array and returns a map from each stored label value to its array position. */ - private static Map readCoordinateIndex(final N5Reader reader, - final String dataset, - final BeamCorrectionParameters beam) { - final double[] values = readCoordinateValues(reader, dataset, beam); + private static Map readCoordinateIndex(final ZarrKeyValueReader reader, + final String dataset) { + final double[] values = readCoordinateValues(reader, dataset); if (values == null) { throw new IllegalArgumentException("coordinate array '" + dataset + "' not found in the zarr container; " + "point --zarrPath at a container that physically contains the scan/slab/sfov coordinate arrays"); @@ -342,12 +373,11 @@ private static Map readCoordinateIndex(final N5Reader reader, } /** Reads a 1D array into an int array indexed by position, or returns null if the dataset is absent. */ - private static int[] readOptionalIntArray(final N5Reader reader, - final String dataset, - final BeamCorrectionParameters beam) { + private static int[] readOptionalIntArray(final ZarrKeyValueReader reader, + final String dataset) { final double[] values; try { - values = readCoordinateValues(reader, dataset, beam); + values = readCoordinateValues(reader, dataset); } catch (final Exception e) { LOG.warn("readOptionalIntArray: could not read {} ({}), slab cross-check will be unavailable", dataset, e.getMessage()); return null; @@ -372,20 +402,16 @@ private static int[] readOptionalIntArray(final N5Reader reader, * that are physically present), the resulting {@link ZarrDatasetAttributes} is built via the reader, and the * chunks are read with {@link N5Reader#readBlock} (which honors the zarr little-endian byte order). */ - private static double[] readCoordinateValues(final N5Reader reader, - final String dataset, - final BeamCorrectionParameters beam) { - final JsonObject zArray = readZArrayJson(dataset, beam); + private static double[] readCoordinateValues(final ZarrKeyValueReader reader, + final String dataset) { + final JsonObject zArray = readDatasetJson(reader, dataset, ZarrKeyValueReader.ZARRAY_FILE); if (zArray == null) { return null; } if (!zArray.has("fill_value") || zArray.get("fill_value").isJsonNull()) { zArray.add("fill_value", new JsonPrimitive("0")); } - if (!(reader instanceof ZarrKeyValueReader)) { - throw new IllegalArgumentException("expected a zarr reader but got " + reader.getClass().getName()); - } - final ZarrDatasetAttributes attributes = ((ZarrKeyValueReader) reader).createDatasetAttributes(zArray); + final ZarrDatasetAttributes attributes = reader.createDatasetAttributes(zArray); if (attributes == null) { throw new IllegalArgumentException("could not parse .zarray for coordinate dataset " + dataset); } @@ -409,35 +435,6 @@ private static double[] readCoordinateValues(final N5Reader reader, return values; } - /** Reads the raw .zarray JSON for a dataset from the per-array file, falling back to consolidated .zmetadata. */ - private static JsonObject readZArrayJson(final String dataset, - final BeamCorrectionParameters beam) { - final Path perArray = Paths.get(beam.zarrPath, dataset, ".zarray"); - try { - if (Files.isRegularFile(perArray)) { - return JsonParser.parseString(Files.readString(perArray)).getAsJsonObject(); - } - } catch (final Exception e) { - LOG.warn("readZArrayJson: failed to read {} ({})", perArray, e.getMessage()); - } - try { - final Path zMetadata = Paths.get(beam.zarrPath, ".zmetadata"); - if (Files.isRegularFile(zMetadata)) { - final JsonObject root = JsonParser.parseString(Files.readString(zMetadata)).getAsJsonObject(); - final JsonObject metadata = root.getAsJsonObject("metadata"); - if (metadata != null) { - final JsonObject zArray = metadata.getAsJsonObject(dataset + "/.zarray"); - if (zArray != null) { - return zArray.deepCopy(); - } - } - } - } catch (final Exception e) { - LOG.warn("readZArrayJson: failed to read .zmetadata for {} ({})", dataset, e.getMessage()); - } - return null; - } - /** Copies a decoded data block into dst[offset...], applying unsigned promotion based on the data type. */ private static void copyBlockValues(final Object data, final DataType dataType, final double[] dst, final int offset) { if (data instanceof byte[]) { diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java index d07bd0eff..f5dd4b7ee 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java @@ -14,9 +14,11 @@ public class BeamCorrectionParameters @Parameter( names = "--zarrPath", - description = "Path to the multi-SEM acquisition zarr (xlog) container " + + description = "Path or URI for the multi-SEM acquisition zarr (xlog) container " + "holding the intensity correction parameters and the scan/slab/sfov coordinate arrays " + - "(e.g. /path/to/xlog_wafer_61.zarr)", required = true) + "(e.g. /groups/hess/hesslab/ibeammsem/system_02/wafers/wafer_61/xlog/xlog_wafer_61.zarr or " + + "gs://janelia-spark-test/library/xlog_wafer_61.zarr)", + required = true) public String zarrPath; @Parameter( From 09b24b304c109bbc665ce19c63783f9d648346da Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Tue, 4 Aug 2026 12:58:42 -0400 Subject: [PATCH 10/40] rename ThomasCalibrationIntensityCorrectionClient to BeamCorrectionClient --- ...rrectionClient.java => BeamCorrectionClient.java} | 8 ++++---- .../spark/multisem/BeamCorrectionSparkClient.java | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) rename render-ws-java-client/src/main/java/org/janelia/render/client/multisem/{ThomasCalibrationIntensityCorrectionClient.java => BeamCorrectionClient.java} (98%) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/BeamCorrectionClient.java similarity index 98% rename from render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java rename to render-ws-java-client/src/main/java/org/janelia/render/client/multisem/BeamCorrectionClient.java index 901d33097..b59a9f53b 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/BeamCorrectionClient.java @@ -76,7 +76,7 @@ * * @author Michael Innerberger */ -public class ThomasCalibrationIntensityCorrectionClient { +public class BeamCorrectionClient { public static class Parameters extends CommandLineParameters { @@ -100,7 +100,7 @@ public void runClient(final String[] args) throws Exception { parameters.beam.validate(); - final ThomasCalibrationIntensityCorrectionClient client = new ThomasCalibrationIntensityCorrectionClient(); + final BeamCorrectionClient client = new BeamCorrectionClient(); final RenderDataClient dataClient = parameters.renderWeb.getDataClient(); client.correctStack(dataClient, parameters.stack, @@ -120,7 +120,7 @@ public void runClient(final String[] args) throws Exception { private static final String B_REF_KEY = "b_ref"; private static final String ZMETADATA_FILE = ".zmetadata"; - public ThomasCalibrationIntensityCorrectionClient() { + public BeamCorrectionClient() { } public void correctStack(final RenderDataClient dataClient, @@ -522,5 +522,5 @@ private static int remainingAxis(final RandomAccessibleInterval rai, final in throw new IllegalArgumentException("could not identify the parameter axis of the correction array"); } - private static final Logger LOG = LoggerFactory.getLogger(ThomasCalibrationIntensityCorrectionClient.class); + private static final Logger LOG = LoggerFactory.getLogger(BeamCorrectionClient.class); } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java index 2e5eac43f..1af8b3de1 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java @@ -12,7 +12,7 @@ import org.janelia.alignment.spec.stack.StackId; import org.janelia.render.client.ClientRunner; import org.janelia.render.client.RenderDataClient; -import org.janelia.render.client.multisem.ThomasCalibrationIntensityCorrectionClient; +import org.janelia.render.client.multisem.BeamCorrectionClient; import org.janelia.render.client.parameter.BeamCorrectionParameters; import org.janelia.render.client.parameter.CommandLineParameters; import org.janelia.render.client.parameter.MultiProjectParameters; @@ -26,7 +26,7 @@ /** * Spark client for applying the pre-computed degree-0 beam-homogenization correction to multi-SEM stacks. * - * @see ThomasCalibrationIntensityCorrectionClient + * @see BeamCorrectionClient */ public class BeamCorrectionSparkClient implements Serializable, AlignmentPipelineStep { @@ -131,10 +131,10 @@ public void correctBeamIntensity(final JavaSparkContext sparkContext, stackId.getOwner(), stackId.getProject()); - new ThomasCalibrationIntensityCorrectionClient().correctStack(executorDataClient, - stackId.getStack(), - beamCorrection, - true); + new BeamCorrectionClient().correctStack(executorDataClient, + stackId.getStack(), + beamCorrection, + true); }); LOG.info("correctBeamIntensity: exit, corrected {} stacks", stackIdList.size()); From 5906b35bf1b37509aa4b929b5b2b401bfec3ba07 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Tue, 4 Aug 2026 15:15:50 -0400 Subject: [PATCH 11/40] add 'other' pipeline stackId naming group and use it for TileIdHackClient and BeamCorrectionSparkClient runs --- .../spec/stack/PipelineStackIdNamingGroups.java | 12 ++++++++++-- .../spark/multisem/BeamCorrectionSparkClient.java | 11 +++++++++-- .../spark/pipeline/AlignmentPipelineParameters.java | 4 ++++ .../render/client/spark/tile/TileIdHackClient.java | 8 ++++---- .../pipeline/AlignmentPipelineParametersTest.java | 10 ++++++++++ .../resources/pipeline/msem_alignment_pipeline.json | 3 +++ 6 files changed, 40 insertions(+), 8 deletions(-) diff --git a/render-app/src/main/java/org/janelia/alignment/spec/stack/PipelineStackIdNamingGroups.java b/render-app/src/main/java/org/janelia/alignment/spec/stack/PipelineStackIdNamingGroups.java index fa63eda48..27f128650 100644 --- a/render-app/src/main/java/org/janelia/alignment/spec/stack/PipelineStackIdNamingGroups.java +++ b/render-app/src/main/java/org/janelia/alignment/spec/stack/PipelineStackIdNamingGroups.java @@ -13,19 +13,22 @@ public class PipelineStackIdNamingGroups private final StackIdNamingGroup raw; private final StackIdNamingGroup aligned; private final StackIdNamingGroup intensityCorrected; + private final StackIdNamingGroup other; /** No-arg constructor required for JSON deserialization. */ @SuppressWarnings("unused") private PipelineStackIdNamingGroups() { - this(null, null, null); + this(null, null, null, null); } public PipelineStackIdNamingGroups(final StackIdNamingGroup raw, final StackIdNamingGroup aligned, - final StackIdNamingGroup intensityCorrected) { + final StackIdNamingGroup intensityCorrected, + final StackIdNamingGroup other) { this.raw = raw; this.aligned = aligned; this.intensityCorrected = intensityCorrected; + this.other = other; } public StackIdNamingGroup getRaw() { @@ -39,4 +42,9 @@ public StackIdNamingGroup getAligned() { public StackIdNamingGroup getIntensityCorrected() { return intensityCorrected; } + + public StackIdNamingGroup getOther() { + return other; + } + } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java index 1af8b3de1..0bee50c8c 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java @@ -10,6 +10,7 @@ import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.janelia.alignment.spec.stack.StackId; +import org.janelia.alignment.spec.stack.StackIdNamingGroup; import org.janelia.render.client.ClientRunner; import org.janelia.render.client.RenderDataClient; import org.janelia.render.client.multisem.BeamCorrectionClient; @@ -92,8 +93,14 @@ public void runPipelineStep(final JavaSparkContext sparkContext, final AlignmentPipelineParameters pipelineParameters) throws IllegalArgumentException, IOException { - final MultiProjectParameters multiProject = - pipelineParameters.getMultiProject(pipelineParameters.getRawNamingGroup()); + final StackIdNamingGroup otherNamingGroup = pipelineParameters.getOtherNamingGroup(); + if (otherNamingGroup == null) { + throw new IllegalArgumentException( + "The " + AlignmentPipelineStepId.CORRECT_BEAM_INTENSITY + " pipeline step requires that " + + "an 'other' pipelineStackGroup is defined in the pipeline parameters."); + } + + final MultiProjectParameters multiProject = pipelineParameters.getMultiProject(otherNamingGroup); correctBeamIntensity(sparkContext, multiProject, diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java index 1aa8f6dd7..0795172b2 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java @@ -143,6 +143,10 @@ public StackIdNamingGroup getIntensityCorrectedNamingGroup() { return pipelineStackGroups == null ? null : pipelineStackGroups.getIntensityCorrected(); } + public StackIdNamingGroup getOtherNamingGroup() { + return pipelineStackGroups == null ? null : pipelineStackGroups.getOther(); + } + public List getPipelineSteps() { return pipelineSteps; } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java index d66f00319..af2ff6c06 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java @@ -88,13 +88,13 @@ public void runPipelineStep(final JavaSparkContext sparkContext, final AlignmentPipelineParameters pipelineParameters) throws IllegalArgumentException, IOException { - final StackIdNamingGroup rawNamingGroup = pipelineParameters.getRawNamingGroup(); - if (rawNamingGroup == null) { + final StackIdNamingGroup otherNamingGroup = pipelineParameters.getOtherNamingGroup(); + if (otherNamingGroup == null) { throw new IllegalArgumentException( "The " + AlignmentPipelineStepId.HACK_TILE_ID + " pipeline step requires that " + - "a raw pipelineStackGroup is defined in the pipeline parameters."); + "an 'other' pipelineStackGroup is defined in the pipeline parameters."); } - final MultiProjectParameters multiProject = pipelineParameters.getMultiProject(rawNamingGroup); + final MultiProjectParameters multiProject = pipelineParameters.getMultiProject(otherNamingGroup); final Parameters clientParameters = new Parameters(multiProject); copyStackAndFixTileIds(sparkContext, clientParameters); } diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParametersTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParametersTest.java index 5ca56fe31..90418e3c6 100644 --- a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParametersTest.java +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParametersTest.java @@ -46,6 +46,16 @@ public void testBuildStepClients() AlignmentPipelineStepId.GENERATE_MIPMAPS, stepClient.getDefaultStepId()); } + @Test + public void testNamingGroupSetup() + throws IOException { + final AlignmentPipelineParameters pipelineParameters = loadTestParameters(); + Assert.assertNotNull("rawNamingGroup is null", pipelineParameters.getRawNamingGroup()); + Assert.assertNotNull("alignedNamingGroup is null", pipelineParameters.getAlignedNamingGroup()); + Assert.assertNotNull("intensityCorrectedNamingGroup is null", pipelineParameters.getIntensityCorrectedNamingGroup()); + Assert.assertNotNull("otherNamingGroup is null", pipelineParameters.getOtherNamingGroup()); + } + @Test public void testLoadParametersFromUrl() throws IOException { diff --git a/render-ws-spark-client/src/test/resources/pipeline/msem_alignment_pipeline.json b/render-ws-spark-client/src/test/resources/pipeline/msem_alignment_pipeline.json index e593724c9..9daa56dec 100644 --- a/render-ws-spark-client/src/test/resources/pipeline/msem_alignment_pipeline.json +++ b/render-ws-spark-client/src/test/resources/pipeline/msem_alignment_pipeline.json @@ -17,6 +17,9 @@ }, "intensityCorrected": { "stackPattern": "^c.*_ic$" + }, + "other": { + "stackPattern": "^w61_s..._r.._gc$" } }, "pipelineSteps": [ From 09dc5834bbdd9ad7673092df674858131d88b332 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Tue, 4 Aug 2026 15:51:10 -0400 Subject: [PATCH 12/40] test tile pairs with excludeDifferentMfovNeighbors true --- .../render/client/TilePairClientTest.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/TilePairClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/TilePairClientTest.java index c5888ec88..39a46226f 100644 --- a/render-ws-java-client/src/test/java/org/janelia/render/client/TilePairClientTest.java +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/TilePairClientTest.java @@ -205,15 +205,19 @@ public TileBoundsRTree buildRTree(final double z) { // public static void main(final String[] args) { // // final String[] effectiveArgs = (args != null) && (args.length > 0) ? args : new String[] { -// "--baseDataUrl", "http://tem-services.int.janelia.org:8080/render-ws/v1", -// "--owner", "flyTEM", -// "--project", "FAFB_montage", -// "--stack", "check_923_split_rough", +// "--baseDataUrl", "http://renderer-dev.int.janelia.org:8080/render-ws/v1", +// "--owner", "hess_wafers_60_61", +// "--project", "w61_serial_140_to_149", +// "--stack", "w61_s140_r00_gc", // "--xyNeighborFactor", "0.6", // "--excludeCornerNeighbors", "false", -// "--excludeSameLayerNeighbors", "true", +// "--excludeSameLayerNeighbors", "false", +// "--excludeSameSectionNeighbors", "false", +// "--excludeSameMfovNeighbors", "false", +// "--excludeDifferentMfovNeighbors", "true", // "--excludeCompletelyObscuredTiles", "false", -// "--zNeighborDistance", "40", +// "--zNeighborDistance", "0", +//// "--z", "1", "2", "3", // "--toJson", "/Users/trautmane/Desktop/test_pairs.json" // }; // TilePairClient.main(effectiveArgs); From 569f19ec8c8659d43034e951d27b95eddc55406b Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Tue, 4 Aug 2026 19:34:07 -0400 Subject: [PATCH 13/40] add overwriteSourceStack parameter to BeamCorrectionParameters --- .../render/client/multisem/BeamCorrectionClient.java | 3 +-- .../client/parameter/BeamCorrectionParameters.java | 12 ++++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/BeamCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/BeamCorrectionClient.java index b59a9f53b..8642fdb53 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/BeamCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/BeamCorrectionClient.java @@ -128,8 +128,7 @@ public void correctStack(final RenderDataClient dataClient, final BeamCorrectionParameters beam, final boolean completeStack) throws IOException { - - final String targetStack = stack + beam.targetStackSuffix; + final String targetStack = beam.getTargetStack(stack); try (final N5Reader reader = new N5Factory().openReader(StorageFormat.ZARR, beam.zarrPath)) { LOG.info("correctStack: opened {} using {}", beam.zarrPath, reader.getClass().getSimpleName()); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java index f5dd4b7ee..f7a068d6b 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java @@ -82,6 +82,13 @@ public class BeamCorrectionParameters description = "Target stack name is the the source stack name with this suffix appended") public String targetStackSuffix = "_bc"; + @Parameter( + names = "--overwriteSourceStack", + description = "Indicates that beam corrected tile specs should be written back to the source stack. " + + "When specified, this parameter will override the targetStackSuffix parameter.", + arity = 0) + public boolean overwriteSourceStack = false; + public BeamCorrectionParameters() { } @@ -108,6 +115,11 @@ public String toString() { ", inverted=" + inverted + ", sfovLabelOffset=" + sfovLabelOffset + ", targetStackSuffix='" + targetStackSuffix + '\'' + + ", overwriteSourceStack=" + overwriteSourceStack + '}'; } + + public String getTargetStack(final String sourceStack) { + return overwriteSourceStack ? sourceStack : sourceStack + targetStackSuffix; + } } \ No newline at end of file From 687900f9693c5513a208cae77e0c670841bf8b1d Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Tue, 4 Aug 2026 19:40:07 -0400 Subject: [PATCH 14/40] switch BeamCorrectionSparkClient and TileIdHackClient back to rawNamingGroup --- .../client/spark/multisem/BeamCorrectionSparkClient.java | 8 ++++---- .../render/client/spark/tile/TileIdHackClient.java | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java index 0bee50c8c..0837d66d5 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java @@ -93,14 +93,14 @@ public void runPipelineStep(final JavaSparkContext sparkContext, final AlignmentPipelineParameters pipelineParameters) throws IllegalArgumentException, IOException { - final StackIdNamingGroup otherNamingGroup = pipelineParameters.getOtherNamingGroup(); - if (otherNamingGroup == null) { + final StackIdNamingGroup rawNamingGroup = pipelineParameters.getRawNamingGroup(); + if (rawNamingGroup == null) { throw new IllegalArgumentException( "The " + AlignmentPipelineStepId.CORRECT_BEAM_INTENSITY + " pipeline step requires that " + - "an 'other' pipelineStackGroup is defined in the pipeline parameters."); + "a 'raw' pipelineStackGroup is defined in the pipeline parameters."); } - final MultiProjectParameters multiProject = pipelineParameters.getMultiProject(otherNamingGroup); + final MultiProjectParameters multiProject = pipelineParameters.getMultiProject(rawNamingGroup); correctBeamIntensity(sparkContext, multiProject, diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java index af2ff6c06..451faec91 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java @@ -88,13 +88,13 @@ public void runPipelineStep(final JavaSparkContext sparkContext, final AlignmentPipelineParameters pipelineParameters) throws IllegalArgumentException, IOException { - final StackIdNamingGroup otherNamingGroup = pipelineParameters.getOtherNamingGroup(); - if (otherNamingGroup == null) { + final StackIdNamingGroup rawNamingGroup = pipelineParameters.getOtherNamingGroup(); + if (rawNamingGroup == null) { throw new IllegalArgumentException( "The " + AlignmentPipelineStepId.HACK_TILE_ID + " pipeline step requires that " + - "an 'other' pipelineStackGroup is defined in the pipeline parameters."); + "a 'raw' pipelineStackGroup is defined in the pipeline parameters."); } - final MultiProjectParameters multiProject = pipelineParameters.getMultiProject(otherNamingGroup); + final MultiProjectParameters multiProject = pipelineParameters.getMultiProject(rawNamingGroup); final Parameters clientParameters = new Parameters(multiProject); copyStackAndFixTileIds(sparkContext, clientParameters); } From e31c6251a69b61108857b6943f330156db4ee95d Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Tue, 4 Aug 2026 20:06:28 -0400 Subject: [PATCH 15/40] switch TileIdHackClient back to rawNamingGroup --- .../org/janelia/render/client/spark/tile/TileIdHackClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java index 451faec91..822db1cbf 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java @@ -88,7 +88,7 @@ public void runPipelineStep(final JavaSparkContext sparkContext, final AlignmentPipelineParameters pipelineParameters) throws IllegalArgumentException, IOException { - final StackIdNamingGroup rawNamingGroup = pipelineParameters.getOtherNamingGroup(); + final StackIdNamingGroup rawNamingGroup = pipelineParameters.getRawNamingGroup(); if (rawNamingGroup == null) { throw new IllegalArgumentException( "The " + AlignmentPipelineStepId.HACK_TILE_ID + " pipeline step requires that " + From 0c3e12825e23fb6131dcede7761f6521c70e31a8 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Wed, 5 Aug 2026 15:16:37 -0400 Subject: [PATCH 16/40] improve list-match-collections.sh output --- render-ws-with-mongo-db/list-match-collections.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/render-ws-with-mongo-db/list-match-collections.sh b/render-ws-with-mongo-db/list-match-collections.sh index d392029c8..738d3d8a6 100755 --- a/render-ws-with-mongo-db/list-match-collections.sh +++ b/render-ws-with-mongo-db/list-match-collections.sh @@ -2,7 +2,9 @@ OWNER="${1:-hess_wafers_60_61}" -printf "\nMatch collections for owner %s\n" "${OWNER}" +printf "\nMatch collections for owner %s are:\n\n" "${OWNER}" URL="http://localhost:8080/render-ws/v1/owner/${OWNER}/matchCollections" -curl -X GET --silent --header 'Accept: application/json' "${URL}" | jq -r '.[] | "\(.collectionId.name)"' | sort \ No newline at end of file +curl -X GET --silent --header 'Accept: application/json' "${URL}" | jq -r '.[] | "\(.collectionId.name)"' | sort + +echo \ No newline at end of file From 2e8a4cc2493b742a3d58a4e835f8d52397bc1b99 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Thu, 6 Aug 2026 09:56:04 -0400 Subject: [PATCH 17/40] replace skipMatchCorrection parameter for creep correction with matchCorrectionType parameter with possible values: OVERWRITE_SOURCE, WRITE_TO_TARGET, or SKIP --- .../parameter/CreepCorrectionParameters.java | 22 +++++++++++++++---- .../multisem/CreepCorrectionSparkClient.java | 12 +++++++--- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java index afc19f31f..7f18defcc 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java @@ -15,6 +15,12 @@ public class CreepCorrectionParameters implements Serializable { + public enum MatchCorrectionType { + OVERWRITE_SOURCE, + WRITE_TO_TARGET, + SKIP + } + @Parameter( names = "--creepTargetStackSuffix", description = "Target stack name is the the source stack name with this suffix appended") @@ -31,9 +37,9 @@ public class CreepCorrectionParameters public Double maxZ; @Parameter( - names = "--skipMatchCorrection", - description = "Skip transforming match coordinates (default is to transform them)") - public boolean skipMatchCorrection = false; + names = "--matchCorrectionType", + description = "Type of match correction") + public MatchCorrectionType matchCorrectionType = MatchCorrectionType.WRITE_TO_TARGET; @Parameter( names = "--parameterCsvDir", @@ -45,6 +51,14 @@ public class CreepCorrectionParameters public CreepCorrectionParameters() { } + public boolean overwriteMatchData() { + return matchCorrectionType == MatchCorrectionType.OVERWRITE_SOURCE; + } + + public boolean writeMatchDataToTargetCollection() { + return matchCorrectionType == MatchCorrectionType.WRITE_TO_TARGET; + } + public void validate() throws IllegalArgumentException { @@ -65,7 +79,7 @@ public String toString() { return "{targetStackSuffix='" + targetStackSuffix + '\'' + ", minZ=" + minZ + ", maxZ=" + maxZ + - ", skipMatchCorrection=" + skipMatchCorrection + + ", matchCorrectionType=" + matchCorrectionType + ", parameterCsvDir='" + parameterCsvDir + '\'' + '}'; } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java index 462751b89..bdb0bdde7 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java @@ -151,8 +151,6 @@ public void correctCreep(final JavaSparkContext sparkContext, final String sourceStack = sourceStackId.getStack(); final String targetStack = creepCorrection.getTargetStack(sourceStack); - final MatchCollectionId targetMatchCollectionId = new MatchCollectionId(matchCollectionId.getOwner(), - targetStack + "_match"); final RenderDataClient sourceDataClient = new RenderDataClient(baseDataUrl, sourceStackId.getOwner(), @@ -214,8 +212,16 @@ public void correctCreep(final JavaSparkContext sparkContext, sourceDataClient.setStackState(targetStack, StackMetaData.StackState.COMPLETE); // Phase 2: transform matches - if (! creepCorrection.skipMatchCorrection) { + if (creepCorrection.overwriteMatchData()) { + + transformMatches(sparkContext, baseDataUrl, matchCollectionId, matchCollectionId, allResults); + + } else if (creepCorrection.writeMatchDataToTargetCollection()) { + + final MatchCollectionId targetMatchCollectionId = new MatchCollectionId(matchCollectionId.getOwner(), + targetStack + "_match"); transformMatches(sparkContext, baseDataUrl, matchCollectionId, targetMatchCollectionId, allResults); + } else { LOG.info("correctCreep: skipping match correction"); } From f5d698de8983e3875017846c8295335bebd295ed Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Thu, 6 Aug 2026 10:21:33 -0400 Subject: [PATCH 18/40] add OVERWRITE_SOURCE_AND_RENAME_AS_TARGET matchCorrectionType --- .../parameter/CreepCorrectionParameters.java | 9 ++++ .../multisem/CreepCorrectionSparkClient.java | 44 +++++++++++++++---- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java index 7f18defcc..d4ac0939a 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java @@ -17,6 +17,7 @@ public class CreepCorrectionParameters public enum MatchCorrectionType { OVERWRITE_SOURCE, + OVERWRITE_SOURCE_AND_RENAME_AS_TARGET, WRITE_TO_TARGET, SKIP } @@ -51,10 +52,18 @@ public enum MatchCorrectionType { public CreepCorrectionParameters() { } + public boolean correctMatches() { + return matchCorrectionType != MatchCorrectionType.SKIP; + } + public boolean overwriteMatchData() { return matchCorrectionType == MatchCorrectionType.OVERWRITE_SOURCE; } + public boolean overwriteMatchDataAndRenameAsTarget() { + return matchCorrectionType == MatchCorrectionType.OVERWRITE_SOURCE_AND_RENAME_AS_TARGET; + } + public boolean writeMatchDataToTargetCollection() { return matchCorrectionType == MatchCorrectionType.WRITE_TO_TARGET; } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java index bdb0bdde7..47787da55 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java @@ -212,15 +212,44 @@ public void correctCreep(final JavaSparkContext sparkContext, sourceDataClient.setStackState(targetStack, StackMetaData.StackState.COMPLETE); // Phase 2: transform matches - if (creepCorrection.overwriteMatchData()) { + if (creepCorrection.correctMatches()) { - transformMatches(sparkContext, baseDataUrl, matchCollectionId, matchCollectionId, allResults); - - } else if (creepCorrection.writeMatchDataToTargetCollection()) { + final RenderDataClient driverMatchClient = new RenderDataClient(baseDataUrl, + matchCollectionId.getOwner(), + matchCollectionId.getName()); final MatchCollectionId targetMatchCollectionId = new MatchCollectionId(matchCollectionId.getOwner(), targetStack + "_match"); - transformMatches(sparkContext, baseDataUrl, matchCollectionId, targetMatchCollectionId, allResults); + + if (creepCorrection.overwriteMatchData()) { + + transformMatches(sparkContext, + driverMatchClient, + baseDataUrl, + matchCollectionId, + matchCollectionId, + allResults); + + } else if (creepCorrection.overwriteMatchDataAndRenameAsTarget()) { + + transformMatches(sparkContext, + driverMatchClient, + baseDataUrl, + matchCollectionId, + matchCollectionId, + allResults); + + driverMatchClient.renameMatchCollection(targetMatchCollectionId); + + } else if (creepCorrection.writeMatchDataToTargetCollection()) { + + transformMatches(sparkContext, + driverMatchClient, + baseDataUrl, + matchCollectionId, + targetMatchCollectionId, + allResults); + } } else { LOG.info("correctCreep: skipping match correction"); @@ -230,16 +259,13 @@ public void correctCreep(final JavaSparkContext sparkContext, } private void transformMatches(final JavaSparkContext sparkContext, + final RenderDataClient driverMatchClient, final String baseDataUrl, final MatchCollectionId matchCollectionId, final MatchCollectionId targetMatchCollectionId, final Map> allResults) throws IOException { - final RenderDataClient driverMatchClient = new RenderDataClient(baseDataUrl, - matchCollectionId.getOwner(), - matchCollectionId.getName()); - final List pGroupIds = driverMatchClient.getMatchPGroupIds(); if (pGroupIds.isEmpty()) { From 6989d587c2595d4684073c85fad25abd6c287ebe Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Thu, 6 Aug 2026 11:10:29 -0400 Subject: [PATCH 19/40] remove duplicated step 1 paragraph in javadoc --- .../client/multisem/MFOVMontageMatchPatchClient.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVMontageMatchPatchClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVMontageMatchPatchClient.java index b4426bfdd..e173efcaf 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVMontageMatchPatchClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVMontageMatchPatchClient.java @@ -50,13 +50,6 @@ * to ensure that standard matches are given precedence. * *

  • - * Step 1: If a sameLayerDerivedMatchWeight is specified and unconnected pairs exist, - * the client loops through each unconnected pair and tries to find existing matches for that same pair in another - * MFOV in the same layer. If found, the existing matches are copied to the unconnected pair and stored - * with the sameLayerDerivedMatchWeight which is typically some reduced value like 0.15 - * to ensure that standard matches are given precedence. - *
    • - *
  • * Step 2: If a crossLayerDerivedMatchWeight is specified and unconnected pairs remain after step 1, * the client collects matches for the unconnected pair in all other layers and fits them to a * "montage patch match model". Montage patch matches are derived by applying the model to each SFOV tile's From 6631383f8973b9aabbdd9382165ba63295ff3c29 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Thu, 6 Aug 2026 15:11:42 -0400 Subject: [PATCH 20/40] add rawNamingGroup check to CreepCorrectionSparkClient --- .../spark/multisem/CreepCorrectionSparkClient.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java index 47787da55..e0d3c2cfd 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java @@ -19,6 +19,7 @@ import org.janelia.alignment.match.MatchCollectionId; import org.janelia.alignment.spec.ResolvedTileSpecCollection; import org.janelia.alignment.spec.stack.StackId; +import org.janelia.alignment.spec.stack.StackIdNamingGroup; import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.alignment.spec.stack.StackWithZValues; import org.janelia.render.client.ClientRunner; @@ -119,9 +120,15 @@ public void validatePipelineParameters(final AlignmentPipelineParameters pipelin public void runPipelineStep(final JavaSparkContext sparkContext, final AlignmentPipelineParameters pipelineParameters) throws IllegalArgumentException, IOException { - - final MultiProjectParameters multiProject = - pipelineParameters.getMultiProject(pipelineParameters.getRawNamingGroup()); + + final StackIdNamingGroup rawNamingGroup = pipelineParameters.getRawNamingGroup(); + if (rawNamingGroup == null) { + throw new IllegalArgumentException( + "The " + AlignmentPipelineStepId.CORRECT_CREEP + " pipeline step requires that " + + "a 'raw' pipelineStackGroup is defined in the pipeline parameters."); + } + + final MultiProjectParameters multiProject = pipelineParameters.getMultiProject(rawNamingGroup); for (final StackWithZValues stackWithAllZ : multiProject.buildListOfStackWithAllZ()) { correctCreep(sparkContext, From 0a945b006a3ce5d3b78661c3e3d83a356c995e7e Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Thu, 6 Aug 2026 15:14:18 -0400 Subject: [PATCH 21/40] change MFOV_AS_TILE naming group from raw to other --- .../client/spark/multisem/MFOVASTileClient.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java index 26d416774..b9f537f06 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java @@ -29,6 +29,7 @@ import org.janelia.alignment.spec.ResolvedTileSpecCollection.TransformApplicationMethod; import org.janelia.alignment.spec.TileSpec; import org.janelia.alignment.spec.stack.StackId; +import org.janelia.alignment.spec.stack.StackIdNamingGroup; import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.alignment.spec.stack.StackWithZValues; import org.janelia.render.client.ClientRunner; @@ -120,7 +121,15 @@ public void runPipelineStep(final JavaSparkContext sparkContext, final AlignmentPipelineParameters pipelineParameters) throws IllegalArgumentException, IOException { final Parameters clientParameters = new Parameters(); - clientParameters.multiProject = pipelineParameters.getMultiProject(pipelineParameters.getRawNamingGroup()); + + final StackIdNamingGroup otherNamingGroup = pipelineParameters.getOtherNamingGroup(); + if (otherNamingGroup == null) { + throw new IllegalArgumentException( + "The " + AlignmentPipelineStepId.MFOV_AS_TILE + " pipeline step requires that " + + "a 'raw' pipelineStackGroup is defined in the pipeline parameters."); + } + + clientParameters.multiProject = pipelineParameters.getMultiProject(otherNamingGroup); clientParameters.mfovAsTile = pipelineParameters.getMfovAsTile(); run(sparkContext, clientParameters); } From 931c5acbbf5a059d5022a57098c86d0348b39b69 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Thu, 6 Aug 2026 15:15:39 -0400 Subject: [PATCH 22/40] change MFOV_AS_TILE naming group from raw to other --- .../janelia/render/client/spark/multisem/MFOVASTileClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java index b9f537f06..9b04343b8 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java @@ -126,7 +126,7 @@ public void runPipelineStep(final JavaSparkContext sparkContext, if (otherNamingGroup == null) { throw new IllegalArgumentException( "The " + AlignmentPipelineStepId.MFOV_AS_TILE + " pipeline step requires that " + - "a 'raw' pipelineStackGroup is defined in the pipeline parameters."); + "an 'other' pipelineStackGroup is defined in the pipeline parameters."); } clientParameters.multiProject = pipelineParameters.getMultiProject(otherNamingGroup); From e38b1359740a859a8faa4dd0bca16b4a78746827 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Thu, 6 Aug 2026 15:50:42 -0400 Subject: [PATCH 23/40] remove unused affine alignment setup from MFOVAsTileClient --- .../render/client/spark/multisem/MFOVASTileClient.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java index 9b04343b8..57aecd591 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java @@ -592,8 +592,6 @@ private static void alignRenderedMfovAsTileStacks(final JavaSparkContext sparkCo final MFOVAsTileParameters mfovAsTile = mfovAsTileStackLists.getMfovAsTile(); final AffineBlockSolverSetup translationSetup = mfovAsTile.buildMfovAffineBlockSolverSetup(MFOVAsTileParameters.SolveType.TRANSLATION); - final AffineBlockSolverSetup affineSetup = - mfovAsTile.buildMfovAffineBlockSolverSetup(MFOVAsTileParameters.SolveType.AFFINE); final boolean deriveMatchCollectionNamesFromProject = false; // use standard stack-based match collection names final String matchSuffix = ""; // without any suffix @@ -613,10 +611,6 @@ private static void alignRenderedMfovAsTileStacks(final JavaSparkContext sparkCo renderedMfovStackWithAllZ, deriveMatchCollectionNamesFromProject, matchSuffix)); - setupList.add(affineSetup.buildPipelineClone(baseDataUrl, - renderedMfovStackWithAllZ, - deriveMatchCollectionNamesFromProject, - matchSuffix)); } } From 0b53d2b12b5b6490c4732214a10e0153ff3d34f4 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Thu, 6 Aug 2026 15:51:44 -0400 Subject: [PATCH 24/40] rename MFOVASTileClient as MFOVAsTileClient --- .../render/client/spark/multisem/LayerAsTileClient.java | 6 +++--- .../{MFOVASTileClient.java => MFOVAsTileClient.java} | 8 ++++---- .../client/spark/pipeline/AlignmentPipelineStepId.java | 4 ++-- .../client/spark/multisem/MfovPrealignTaskTest.java | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) rename render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/{MFOVASTileClient.java => MFOVAsTileClient.java} (99%) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/LayerAsTileClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/LayerAsTileClient.java index 559baaafe..e5edbe58c 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/LayerAsTileClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/LayerAsTileClient.java @@ -154,7 +154,7 @@ private static void buildDynamicLayerAsTileStacks(final JavaSparkContext sparkCo final List align2DSfovStacksWithAllZ = layerAsTileStackLists.getAlign2DSfovStacksWithAllZ(); - final int parallelism = Math.min(MFOVASTileClient.MAX_PARTITIONS_FOR_ONE_WEB_SERVER, align2DSfovStacksWithAllZ.size()); + final int parallelism = Math.min(MFOVAsTileClient.MAX_PARTITIONS_FOR_ONE_WEB_SERVER, align2DSfovStacksWithAllZ.size()); LOG.info("buildDynamicLayerAsTileStacks: entry, distributing build of {} stack(s) with parallelism {} (defaultParallelism={})", align2DSfovStacksWithAllZ.size(), parallelism, sparkContext.defaultParallelism()); @@ -252,7 +252,7 @@ private static void buildRenderedLayerAsTileStacks(final JavaSparkContext sparkC if (! layerClientInfoList.isEmpty()) { - final int parallelism = Math.min(MFOVASTileClient.MAX_PARTITIONS_FOR_ONE_WEB_SERVER, layerClientInfoList.size()); + final int parallelism = Math.min(MFOVAsTileClient.MAX_PARTITIONS_FOR_ONE_WEB_SERVER, layerClientInfoList.size()); LOG.info("buildRenderedLayerAsTileStacks: distributing rendering for {} layers with parallelism {} (defaultParallelism={})", layerClientInfoList.size(), parallelism, sparkContext.defaultParallelism()); @@ -409,7 +409,7 @@ private static void buildAlign3DSfovStacks(final JavaSparkContext sparkContext, if (! rawSfovStacksNeedingAlign3DStack.isEmpty()) { - final int parallelism = Math.min(MFOVASTileClient.MAX_PARTITIONS_FOR_ONE_WEB_SERVER, rawSfovStacksNeedingAlign3DStack.size()); + final int parallelism = Math.min(MFOVAsTileClient.MAX_PARTITIONS_FOR_ONE_WEB_SERVER, rawSfovStacksNeedingAlign3DStack.size()); LOG.info("buildAlign3DSfovStacks: distributing build of {} stack(s) with parallelism {} (defaultParallelism={})", rawSfovStacksNeedingAlign3DStack.size(), parallelism, sparkContext.defaultParallelism()); diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVAsTileClient.java similarity index 99% rename from render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java rename to render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVAsTileClient.java index 57aecd591..594d78034 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVAsTileClient.java @@ -61,7 +61,7 @@ /** * Spark client for ... */ -public class MFOVASTileClient +public class MFOVAsTileClient implements Serializable, AlignmentPipelineStep { public static class Parameters extends CommandLineParameters { @@ -88,7 +88,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); parameters.parse(args); - final MFOVASTileClient client = new MFOVASTileClient(); + final MFOVAsTileClient client = new MFOVAsTileClient(); client.createContextAndRun(parameters); } }; @@ -96,7 +96,7 @@ public void runClient(final String[] args) throws Exception { } /** Empty constructor required for alignment pipeline steps. */ - public MFOVASTileClient() { + public MFOVAsTileClient() { } /** Create a spark context and run the client with the specified parameters. */ @@ -842,5 +842,5 @@ public int renderTiles() public static final int MAX_PARTITIONS_FOR_ONE_WEB_SERVER = 100000; - private static final Logger LOG = LoggerFactory.getLogger(MFOVASTileClient.class); + private static final Logger LOG = LoggerFactory.getLogger(MFOVAsTileClient.class); } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java index ed354cbc4..b09ef2915 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java @@ -11,7 +11,7 @@ import org.janelia.render.client.spark.multisem.BeamCorrectionSparkClient; import org.janelia.render.client.spark.multisem.CreepCorrectionSparkClient; import org.janelia.render.client.spark.multisem.LayerAsTileClient; -import org.janelia.render.client.spark.multisem.MFOVASTileClient; +import org.janelia.render.client.spark.multisem.MFOVAsTileClient; import org.janelia.render.client.spark.multisem.MFOVMontageMatchPatchClient; import org.janelia.render.client.spark.multisem.UnconnectedCrossMFOVClient; import org.janelia.render.client.spark.newsolver.DistributedAffineBlockSolverClient; @@ -40,7 +40,7 @@ public enum AlignmentPipelineStepId { HACK_TILE_ID(TileIdHackClient::new), RENDER_SCAPE_IMAGES(ScapeClient::new), RENDER_TILES(RenderTilesClient::new), - MFOV_AS_TILE(MFOVASTileClient::new), + MFOV_AS_TILE(MFOVAsTileClient::new), LAYER_AS_TILE(LayerAsTileClient::new); private final Supplier stepClientSupplier; diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/multisem/MfovPrealignTaskTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/multisem/MfovPrealignTaskTest.java index 9378bc809..6988cafae 100644 --- a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/multisem/MfovPrealignTaskTest.java +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/multisem/MfovPrealignTaskTest.java @@ -80,8 +80,8 @@ public static void main(final String[] args) throws Exception { final StackId renderedMfovStackId = dynamicMfovStackId.withStackSuffix(mfovAsTile.getRenderedMfovStackSuffix()); final String runTimestamp = new TileRenderParameters().getRunTimestamp(); - final MFOVASTileClient.JavaRenderTilesClientInfoForLayerMfov z17Info = - new MFOVASTileClient.JavaRenderTilesClientInfoForLayerMfov(baseDataUrl, + final MFOVAsTileClient.JavaRenderTilesClientInfoForLayerMfov z17Info = + new MFOVAsTileClient.JavaRenderTilesClientInfoForLayerMfov(baseDataUrl, dynamicMfovStackId, z17LayerMfov, mfovAsTile, @@ -89,8 +89,8 @@ public static void main(final String[] args) throws Exception { z17Info.setupHackStackAndStorage(); z17Info.renderTiles(); - final MFOVASTileClient.JavaRenderTilesClientInfoForLayerMfov z18Info = - new MFOVASTileClient.JavaRenderTilesClientInfoForLayerMfov(baseDataUrl, + final MFOVAsTileClient.JavaRenderTilesClientInfoForLayerMfov z18Info = + new MFOVAsTileClient.JavaRenderTilesClientInfoForLayerMfov(baseDataUrl, dynamicMfovStackId, z18LayerMfov, mfovAsTile, From d7be26c4b893c223fdaa682e11ad099547f9eba8 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Fri, 7 Aug 2026 16:00:23 -0400 Subject: [PATCH 25/40] add removeMatchPairsWithDifferentMFOVs method --- .../spec/ResolvedTileSpecsWithMatchPairs.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java b/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java index 3fe215846..ac7e15d1f 100644 --- a/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java +++ b/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java @@ -12,6 +12,7 @@ import org.janelia.alignment.json.JsonUtils; import org.janelia.alignment.match.CanvasMatches; +import org.janelia.alignment.multisem.MultiSemUtilities; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -159,6 +160,29 @@ public void normalize(final Integer maxZDistance, this.matchPairs = normalizedMatchPairs; } + /** + * Remove any match pairs where the pTile is in a different MFOV from the qTile. + */ + public void removeMatchPairsWithDifferentMFOVs() { + + final Set sameMFOVMatchPairs = new HashSet<>(matchPairs.size()); + + for (final CanvasMatches pair : matchPairs) { + final TileSpec pTileSpec = resolvedTileSpecs.getTileSpec(pair.getpId()); + final TileSpec qTileSpec = resolvedTileSpecs.getTileSpec(pair.getqId()); + if ((pTileSpec != null) && (qTileSpec != null)) { + final String pMFOV = MultiSemUtilities.getSimpleMfovForTileId(pTileSpec.getTileId()); + final String qMFOV = MultiSemUtilities.getSimpleMfovForTileId(qTileSpec.getTileId()); + if (pMFOV.equals(qMFOV)) { + sameMFOVMatchPairs.add(pair); + } + } + } + + // pairs from web service are not sorted, so sort here to make usage loops more intuitive + this.matchPairs = sameMFOVMatchPairs.stream().sorted().collect(Collectors.toList()); + } + public static ResolvedTileSpecsWithMatchPairs fromJson(final Reader json) { return JSON_HELPER.fromJson(json); } From 37372f7b9ec6303475db9ab2a8c499047bb14246 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Fri, 7 Aug 2026 17:13:39 -0400 Subject: [PATCH 26/40] fix canvas groupId bug --- .../render/client/spark/multisem/MfovPrealignTask.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MfovPrealignTask.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MfovPrealignTask.java index 06910d8ec..bbc45fece 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MfovPrealignTask.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MfovPrealignTask.java @@ -191,8 +191,8 @@ private List generateTilePairs(final ResolvedTileSpecCollec final Rectangle2D rectJ = new Rectangle2D.Double(tj.getMinX(), tj.getMinY(), tj.getWidth(), tj.getHeight()); if (rectI.intersects(rectJ)) { - final CanvasId canvasIdI = new CanvasId(ti.getGroupId(), ti.getTileId()); - final CanvasId canvasIdJ = new CanvasId(tj.getGroupId(), tj.getTileId()); + final CanvasId canvasIdI = new CanvasId(ti.getZ().toString(), ti.getTileId()); + final CanvasId canvasIdJ = new CanvasId(tj.getZ().toString(), tj.getTileId()); pairs.add(new OrderedCanvasIdPair(canvasIdI, canvasIdJ, 0.0)); } } From a80113a3f831c3196ffe1beeafc051b57e19706f Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Sun, 9 Aug 2026 10:12:10 -0400 Subject: [PATCH 27/40] add deriveSfovMatchData to MFOVAsTileParameters --- .../client/parameter/MFOVAsTileParameters.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVAsTileParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVAsTileParameters.java index 93a8d985a..753f3af5b 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVAsTileParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVAsTileParameters.java @@ -36,6 +36,7 @@ public class MFOVAsTileParameters private final String alignedMfovStackSuffix; private final String roughSfovStackSuffix; private final Double minCrossMatchPixelDistance; + private final boolean deriveSfovMatchData; public MFOVAsTileParameters() { this(null, @@ -62,7 +63,8 @@ public MFOVAsTileParameters(final Double mfovRenderScale, null, alignedMfovStackSuffix, roughSfovStackSuffix, - null); + null, + true); } public MFOVAsTileParameters(final Double mfovRenderScale, @@ -73,7 +75,8 @@ public MFOVAsTileParameters(final Double mfovRenderScale, final String renderedMfovRunTimestamp, final String alignedMfovStackSuffix, final String roughSfovStackSuffix, - final Double minCrossMatchPixelDistance) { + final Double minCrossMatchPixelDistance, + final boolean deriveSfovMatchData) { this.mfovRenderScale = mfovRenderScale; this.mfovRootDirectory = mfovRootDirectory; this.prealignedSfovStackSuffix = prealignedSfovStackSuffix; @@ -83,6 +86,7 @@ public MFOVAsTileParameters(final Double mfovRenderScale, this.alignedMfovStackSuffix = alignedMfovStackSuffix; this.roughSfovStackSuffix = roughSfovStackSuffix; this.minCrossMatchPixelDistance = minCrossMatchPixelDistance; + this.deriveSfovMatchData = deriveSfovMatchData; } public Double getMfovRenderScale() { @@ -117,6 +121,10 @@ public Double getMinCrossMatchPixelDistance() { return minCrossMatchPixelDistance; } + public boolean isDeriveSfovMatchData() { + return deriveSfovMatchData; + } + public String getDynamicMfovStackSuffixForRawSfovStack() { return prealignedSfovStackSuffix + dynamicMfovStackSuffix; } From e1effd300203685a246c04437f4f141e194f2122 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Sun, 9 Aug 2026 20:22:14 -0400 Subject: [PATCH 28/40] add applyOffsets method to Matches --- .../java/org/janelia/alignment/match/Matches.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/render-app/src/main/java/org/janelia/alignment/match/Matches.java b/render-app/src/main/java/org/janelia/alignment/match/Matches.java index 759e574d8..c96f7d286 100644 --- a/render-app/src/main/java/org/janelia/alignment/match/Matches.java +++ b/render-app/src/main/java/org/janelia/alignment/match/Matches.java @@ -139,6 +139,19 @@ public ArrayList createPointMatches() { return matches; } + public void applyOffsets(final double pxOffset, + final double pyOffset, + final double qxOffset, + final double qyOffset) { + // NOTE: point coordinates are stored by dimension, so p[0] holds all x values and p[1] holds all y values + for (int i = 0; i < w.length; ++i) { + p[0][i] += pxOffset; + p[1][i] += pyOffset; + q[0][i] += qxOffset; + q[1][i] += qyOffset; + } + } + // NOTE: Convenience getter methods for derived data should be excluded from the JSON model - hence the JsonIgnore annotation. // Newer Jackson versions (e.g. 2.9.6) fail to parse RealPoint with an IllegalArgumentException: // Conflicting setter definitions for property "position": net.imglib2.RealPoint#setPosition(1 params) vs net.imglib2.RealPoint#setPosition(1 params) From 7b7e29b8018704fe794ffe630623cbc084f2cf28 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Sun, 9 Aug 2026 20:24:47 -0400 Subject: [PATCH 29/40] add changeMatchesToWorldCoordinates method and change removeMatchPairsWithDifferentMFOVs method to removeMatchPairsThatReferenceTilesOutsideThisCollection --- .../spec/ResolvedTileSpecsWithMatchPairs.java | 53 +++++++++++++++---- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java b/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java index ac7e15d1f..9a621f3e6 100644 --- a/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java +++ b/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java @@ -12,7 +12,6 @@ import org.janelia.alignment.json.JsonUtils; import org.janelia.alignment.match.CanvasMatches; -import org.janelia.alignment.multisem.MultiSemUtilities; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -161,26 +160,60 @@ public void normalize(final Integer maxZDistance, } /** - * Remove any match pairs where the pTile is in a different MFOV from the qTile. + * Remove any match pairs where the pTile and/or the qTile do not exist in this collection's resolvedTileSpecs. */ - public void removeMatchPairsWithDifferentMFOVs() { + public void removeMatchPairsThatReferenceTilesOutsideThisCollection() { - final Set sameMFOVMatchPairs = new HashSet<>(matchPairs.size()); + final Set updatedMatchPairs = new HashSet<>(matchPairs.size()); for (final CanvasMatches pair : matchPairs) { + if (resolvedTileSpecs.hasTileSpec(pair.getpId()) && + resolvedTileSpecs.hasTileSpec(pair.getqId())) { + updatedMatchPairs.add(pair); + } + } + + // pairs from web service are not sorted, so sort here to make usage loops more intuitive + this.matchPairs = updatedMatchPairs.stream().sorted().collect(Collectors.toList()); + } + + /** + * Change all match points to world coordinates using tile min x and y as offset. + * + * @throws IllegalStateException + * if any match pair identifies a tile spec that is missing from this collection. + */ + public void changeMatchesToWorldCoordinates() + throws IllegalStateException { + + for (final CanvasMatches pair : matchPairs) { + final TileSpec pTileSpec = resolvedTileSpecs.getTileSpec(pair.getpId()); final TileSpec qTileSpec = resolvedTileSpecs.getTileSpec(pair.getqId()); + if ((pTileSpec != null) && (qTileSpec != null)) { - final String pMFOV = MultiSemUtilities.getSimpleMfovForTileId(pTileSpec.getTileId()); - final String qMFOV = MultiSemUtilities.getSimpleMfovForTileId(qTileSpec.getTileId()); - if (pMFOV.equals(qMFOV)) { - sameMFOVMatchPairs.add(pair); + + pair.getMatches().applyOffsets(pTileSpec.getMinX(), + pTileSpec.getMinY(), + qTileSpec.getMinX(), + qTileSpec.getMinY()); + + } else { + + final StringBuilder sb = new StringBuilder(); + if (pTileSpec == null) { + if (qTileSpec == null) { + sb.append("s ").append(pair.getpId()).append(" and ").append(pair.getqId()); + } + sb.append(" ").append(pair.getpId()); + } else { + sb.append(" ").append(pair.getqId()); } + throw new IllegalStateException("missing tile" + sb + " for match pair " + pair.toKeyString()); + } } - // pairs from web service are not sorted, so sort here to make usage loops more intuitive - this.matchPairs = sameMFOVMatchPairs.stream().sorted().collect(Collectors.toList()); } public static ResolvedTileSpecsWithMatchPairs fromJson(final Reader json) { From 84b40ec1fcd46442abf3430c7b65a71e4e025b8b Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Sun, 9 Aug 2026 20:42:45 -0400 Subject: [PATCH 30/40] Update the MFOV pre-alignment step to read matches from render's web service since we generate that match data in an earlier intensity correction step. The MFOVAsTileParameters deriveSfovMatchData field determines whether matches are simply read or are generated during the pre-alignment process. In "small" tests using 5 Google nodes with two w61_s140_r00 z layers, the run with deriveSfovMatchData=false took 61 minutes and the run with deriveSfovMatchData=true took 79 minutes. --- .../spark/multisem/MFOVAsTileClient.java | 32 ++- .../spark/multisem/MfovPrealignTask.java | 230 +++++++++++++----- .../spark/multisem/MfovPrealignTaskTest.java | 39 ++- 3 files changed, 227 insertions(+), 74 deletions(-) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVAsTileClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVAsTileClient.java index 594d78034..79dab3ac7 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVAsTileClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVAsTileClient.java @@ -151,7 +151,9 @@ private void run(final JavaSparkContext sparkContext, clientParameters.multiProject, clientParameters.mfovAsTile); if (clientParameters.mfovAsTile.doPrealign()) { - alignAndIntensityCorrectMfovAsTileStacks(sparkContext, mfovAsTileStackLists); + alignAndIntensityCorrectMfovAsTileStacks(sparkContext, + mfovAsTileStackLists, + clientParameters.mfovAsTile.isDeriveSfovMatchData()); } buildDynamicMfovAsTileStacks(sparkContext, mfovAsTileStackLists); @@ -172,8 +174,10 @@ private void run(final JavaSparkContext sparkContext, } private static void alignAndIntensityCorrectMfovAsTileStacks(final JavaSparkContext sparkContext, - final MFOVAsTileStackLists mfovAsTileStackLists) + final MFOVAsTileStackLists mfovAsTileStackLists, + final boolean deriveSfovMatchData) throws IOException { + LOG.info("alignAndIntensityCorrectMfovAsTileStacks: entry"); final String baseDataUrl = mfovAsTileStackLists.getBaseDataUrl(); @@ -186,7 +190,10 @@ private static void alignAndIntensityCorrectMfovAsTileStacks(final JavaSparkCont for (final StackWithZValues rawSfovStackWithZ : rawSfovStacksWithAllZ) { final StackId rawSfovStackId = rawSfovStackWithZ.getStackId(); + final String rawSfovStack = rawSfovStackId.getStack(); final StackId prealignedStackId = rawSfovStackId.withStackSuffix(prealignedSfovStackSuffix); + final MatchCollectionId rawSfovMatchCollectionId = deriveSfovMatchData ? null : new MatchCollectionId(rawSfovStackId.getOwner(), + rawSfovStack + "_match"); if (mfovAsTileStackLists.isExistingStack(prealignedStackId)) { LOG.info("alignAndIntensityCorrectMfovAsTileStacks: skipping build of {} because it already exists", @@ -195,23 +202,24 @@ private static void alignAndIntensityCorrectMfovAsTileStacks(final JavaSparkCont } // Create prealigned stack - final RenderDataClient dataClient = new RenderDataClient(baseDataUrl, - rawSfovStackId.getOwner(), - rawSfovStackId.getProject()); - final StackMetaData rawStackMetaData = dataClient.getStackMetaData(rawSfovStackId.getStack()); - dataClient.setupDerivedStack(rawStackMetaData, prealignedStackId.getStack()); + final RenderDataClient stackDataClient = new RenderDataClient(baseDataUrl, + rawSfovStackId.getOwner(), + rawSfovStackId.getProject()); + final StackMetaData rawStackMetaData = stackDataClient.getStackMetaData(rawSfovStackId.getStack()); + stackDataClient.setupDerivedStack(rawStackMetaData, prealignedStackId.getStack()); prealignedStackIds.add(prealignedStackId); // Collect MFOV tasks for each layer for (final Double z : rawSfovStackWithZ.getzValues()) { - final List mfovNames = MultiProjectParameters.getSortedMFOVNamesForOneLayer(dataClient, - rawSfovStackId.getStack(), + final List mfovNames = MultiProjectParameters.getSortedMFOVNamesForOneLayer(stackDataClient, + rawSfovStack, z); for (final String mfovName : mfovNames) { mfovTasks.add(new MfovPrealignTask(baseDataUrl, - rawSfovStackId, - prealignedStackId, - new LayerMFOV(z, mfovName)) + rawSfovStackId, + prealignedStackId, + new LayerMFOV(z, mfovName), + rawSfovMatchCollectionId) ); } } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MfovPrealignTask.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MfovPrealignTask.java index bbc45fece..b011487e5 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MfovPrealignTask.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MfovPrealignTask.java @@ -10,6 +10,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import mpicbg.imagefeatures.Feature; import mpicbg.models.ErrorStatistic; @@ -28,14 +29,18 @@ import org.janelia.alignment.match.CanvasFeatureMatcher; import org.janelia.alignment.match.CanvasId; import org.janelia.alignment.match.CanvasMatchResult; +import org.janelia.alignment.match.CanvasMatches; +import org.janelia.alignment.match.MatchCollectionId; import org.janelia.alignment.match.MatchFilter; import org.janelia.alignment.match.ModelType; import org.janelia.alignment.match.OrderedCanvasIdPair; import org.janelia.alignment.match.parameters.FeatureExtractionParameters; import org.janelia.alignment.match.parameters.MatchDerivationParameters; import org.janelia.alignment.multisem.LayerMFOV; +import org.janelia.alignment.spec.Bounds; import org.janelia.alignment.spec.LeafTransformSpec; import org.janelia.alignment.spec.ResolvedTileSpecCollection; +import org.janelia.alignment.spec.ResolvedTileSpecsWithMatchPairs; import org.janelia.alignment.spec.TileSpec; import org.janelia.alignment.spec.stack.StackId; import org.janelia.alignment.util.ImageProcessorCache; @@ -58,6 +63,7 @@ public class MfovPrealignTask implements Serializable { private final StackId rawSfovStackId; private final StackId prealignedStackId; private final LayerMFOV layerMfov; + private final MatchCollectionId matchCollectionId; private static final int CLIP_SIZE = 150; @@ -65,12 +71,14 @@ public MfovPrealignTask( final String baseDataUrl, final StackId rawSfovStackId, final StackId prealignedStackId, - final LayerMFOV layerMfov + final LayerMFOV layerMfov, + final MatchCollectionId matchCollectionId ) { this.baseDataUrl = baseDataUrl; this.rawSfovStackId = rawSfovStackId; this.prealignedStackId = prealignedStackId; this.layerMfov = layerMfov; + this.matchCollectionId = matchCollectionId; } public String toString() { @@ -94,31 +102,22 @@ public void run() rawSfovStackId.getOwner(), rawSfovStackId.getProject()); - // 1. Fetch tile specs for all SFOVs in this MFOV - final ResolvedTileSpecCollection mfovTiles = fetchMfovTileSpecs(dataClient); - - if (mfovTiles.getTileCount() == 0) { - throw new IOException("no tile specs found for " + this); - } - - // 2. Generate tile pairs for matching - final List tilePairs = generateTilePairs(mfovTiles); - - if (tilePairs.isEmpty()) { - LOG.info("run: exit, {}, no tile pairs generated", this); - return; - } - // Initialize the image processor cache with ~1Gb size to hold all tiles in memory - final ImageProcessorCache cache = new ImageProcessorCache(1_000_000_000L, false, false); + final ImageProcessorCache cache = new ImageProcessorCache(1_000_000_000L, + false, + false); - // 3. Align tiles within this MFOV - final ResolvedTileSpecCollection alignedTiles = alignTiles(mfovTiles, tilePairs, cache); + final ResolvedTileSpecsWithMatchPairs sfovTilesAndMatchesForMfov = fetchSfovTileSpecsWithMatchPairs(dataClient, + cache); + final ResolvedTileSpecCollection alignedTiles = alignTiles(sfovTilesAndMatchesForMfov); - // 4. Perform intensity correction - final ResolvedTileSpecCollection alignedIcTiles = intensityCorrectTiles(alignedTiles, tilePairs, cache); + final List tilePairs = sfovTilesAndMatchesForMfov.getMatchPairs().stream() + .map(CanvasMatches::toOrderedPair) + .collect(Collectors.toList()); + final ResolvedTileSpecCollection alignedIcTiles = intensityCorrectTiles(alignedTiles, + tilePairs, + cache); - // 5. Push the aligned tile specs to the prealigned stack dataClient.saveResolvedTiles(alignedIcTiles, prealignedStackId.getStack(), layerMfov.getZ()); final long elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000; @@ -165,13 +164,73 @@ private void setupLogging() { } /** - * Fetch tile specs for all SFOVs belonging to this MFOV at the specified z layer. + * Fetch tile specs for all SFOVs belonging to this MFOV at the specified z layer + * and fetch (or build) the corresponding matches. */ - private ResolvedTileSpecCollection fetchMfovTileSpecs(final RenderDataClient dataClient) throws IOException { + private ResolvedTileSpecsWithMatchPairs fetchSfovTileSpecsWithMatchPairs(final RenderDataClient dataClient, + final ImageProcessorCache cache) + throws IOException { + + LOG.info("fetchSfovTileSpecsWithMatchPairs: entry, {}", this); + final String matchPattern = "_" + layerMfov.getSimpleMfovName() + "_"; // limit to tiles in this MFOV - return dataClient.getResolvedTiles(rawSfovStackId.getStack(), - layerMfov.getZ(), - matchPattern); + + final ResolvedTileSpecsWithMatchPairs resolvedTileSpecsWithMatchPairs; + if (matchCollectionId == null) { + + final ResolvedTileSpecCollection tiles = dataClient.getResolvedTiles(rawSfovStackId.getStack(), + layerMfov.getZ(), + matchPattern); + if (tiles.getTileCount() == 0) { + throw new IOException("no tile specs found for " + this); + } + + final List tilePairs = generateTilePairs(tiles); + if (tilePairs.isEmpty()) { + throw new IOException("no tile pairs generated for " + this); + } + + resolvedTileSpecsWithMatchPairs = generateAndAddMatches(tiles, + tilePairs, + cache); + } else { + + final Double z = layerMfov.getZ(); + final Bounds layerBounds = new Bounds(-Double.MAX_VALUE, -Double.MAX_VALUE, z, + Double.MAX_VALUE, Double.MAX_VALUE, z); + + resolvedTileSpecsWithMatchPairs = dataClient.getResolvedTilesWithMatchPairs(rawSfovStackId.getStack(), + layerBounds, + matchCollectionId.getName(), + null, + matchPattern, + false); + if (resolvedTileSpecsWithMatchPairs.getResolvedTileSpecs().getTileCount() == 0) { + throw new IOException("no tile specs found for " + this); + } + + final int originalMatchPairCount = resolvedTileSpecsWithMatchPairs.getMatchPairCount(); + resolvedTileSpecsWithMatchPairs.removeMatchPairsThatReferenceTilesOutsideThisCollection(); + final int normalizedMatchPairCount = resolvedTileSpecsWithMatchPairs.getMatchPairCount(); + if (normalizedMatchPairCount < originalMatchPairCount) { + LOG.info("fetchSfovTileSpecsWithMatchPairs: removed {} pairs and kept {} pairs for {}", + (originalMatchPairCount - normalizedMatchPairCount), normalizedMatchPairCount, this); + } + + // change matches to world coordinates to be consistent with original manual derivation approach + resolvedTileSpecsWithMatchPairs.changeMatchesToWorldCoordinates(); + } + + if (resolvedTileSpecsWithMatchPairs.getMatchPairCount() == 0) { + throw new IOException("no matches were found or built for " + this); + } + + LOG.info("fetchSfovTileSpecsWithMatchPairs: {} returning {} tiles and {} match pairs", + this, + resolvedTileSpecsWithMatchPairs.getResolvedTileSpecs().getTileCount(), + resolvedTileSpecsWithMatchPairs.getMatchPairCount()); + + return resolvedTileSpecsWithMatchPairs; } /** @@ -204,56 +263,104 @@ private List generateTilePairs(final ResolvedTileSpecCollec /** * Perform SIFT feature matching on the tile pairs. */ - private ResolvedTileSpecCollection alignTiles( - final ResolvedTileSpecCollection tiles, - final List tilePairs, - final ImageProcessorCache cache - ) { + private ResolvedTileSpecsWithMatchPairs generateAndAddMatches(final ResolvedTileSpecCollection tiles, + final List tilePairs, + final ImageProcessorCache cache) { final long startTime = System.currentTimeMillis(); - LOG.info("alignTiles: entry, {}", this); + + LOG.info("generateAndAddMatches: entry, {}", this); + + final List matchPairs = new ArrayList<>(); // Extract features from all mfov tiles final CanvasFeatureExtractor featureExtractor = CanvasFeatureExtractor.build(FEATURE_EXTRACTION_PARAMETERS); final Map> mfovFeatures = new HashMap<>(tiles.getTileCount()); - final Map> modelTiles = new HashMap<>(); for (final TileSpec tileSpec : tiles.getTileSpecs()) { final List tileFeatures = extractBoundaryFeatures(cache, tileSpec, featureExtractor); - mfovFeatures.put(tileSpec.getTileId(), tileFeatures); - modelTiles.put(tileSpec.getTileId(), new Tile<>(new TranslationModel2D())); } // Match features between tile pairs - final CanvasFeatureMatcher featureMatcher = new CanvasFeatureMatcher(MATCH_DERIVATION_PARAMETERS, 1.0); + final double renderScale = 1.0; + final CanvasFeatureMatcher featureMatcher = new CanvasFeatureMatcher(MATCH_DERIVATION_PARAMETERS, + renderScale); + CanvasMatchResult matchResult; for (final OrderedCanvasIdPair pair : tilePairs) { - final String tileIdI = pair.getP().getId(); - final String tileIdJ = pair.getQ().getId(); - final CanvasMatchResult matchResult = featureMatcher.deriveMatchResult( - mfovFeatures.get(tileIdI), - mfovFeatures.get(tileIdJ) - ); + final CanvasId p = pair.getP(); + final CanvasId q = pair.getQ(); - // Connect tiles for optimization... - final Tile modelTileI = modelTiles.get(tileIdI); - final Tile modelTileJ = modelTiles.get(tileIdJ); + final String pTileId = p.getId(); + final String qTileId = q.getId(); + + matchResult = featureMatcher.deriveMatchResult(mfovFeatures.get(pTileId), + mfovFeatures.get(qTileId)); + + // create fake connection if no inliers found if (matchResult == null || matchResult.getTotalNumberOfInliers() == 0) { - // ... with fake matches if no inliers found - final PointMatch fakeMatch = new PointMatch( - new Point(new double[] {0.0, 0.0}), new Point(new double[] {0.0, 0.0}), 1e-4 - ); - modelTileI.connect(modelTileJ, Collections.singletonList(fakeMatch)); - } else { - // ... with real matches if inliers found - final List pointMatches = matchResult.getInlierPointMatchList(); - modelTileI.connect(modelTileJ, pointMatches); + final PointMatch fakeMatch = new PointMatch(new Point(new double[]{0.0, 0.0}), + new Point(new double[]{0.0, 0.0}), + 1e-4); + final List fakeList = new ArrayList<>(); + fakeList.add(fakeMatch); + final List> fakeInliers = new ArrayList<>(); + fakeInliers.add(fakeList); + matchResult = new CanvasMatchResult(null, fakeInliers, 1); } + + final List canvasMatchesList = matchResult.getInlierMatchesList(p.getGroupId(), + p.getId(), + q.getGroupId(), + q.getId(), + renderScale, + CanvasId.ZERO_OFFSETS, + CanvasId.ZERO_OFFSETS); + matchPairs.add(canvasMatchesList.get(0)); } - LOG.info("alignTiles: start optimization, {}", this); + final long elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000; + + LOG.info("generateAndAddMatches: exit, {}, elapsedSeconds={}", this, elapsedSeconds); + + return new ResolvedTileSpecsWithMatchPairs(tiles, matchPairs); + } + + /** + * Align tile pairs. + */ + private ResolvedTileSpecCollection alignTiles(final ResolvedTileSpecsWithMatchPairs sfovTilesAndMatchesForMfov) { + + final long startTime = System.currentTimeMillis(); + LOG.info("alignTiles: entry, {}", this); + + final ResolvedTileSpecCollection tiles = sfovTilesAndMatchesForMfov.getResolvedTileSpecs(); + + final Map> modelTiles = new HashMap<>(); + + for (final TileSpec tileSpec : sfovTilesAndMatchesForMfov.getResolvedTileSpecs().getTileSpecs()) { + modelTiles.put(tileSpec.getTileId(), new Tile<>(new TranslationModel2D())); + } + + int connectedPairCount = 0; + for (final CanvasMatches canvasMatches : sfovTilesAndMatchesForMfov.getMatchPairs()) { + // Connect tiles for optimization... + final Tile pModelTile = modelTiles.get(canvasMatches.getpId()); + final Tile qModelTile = modelTiles.get(canvasMatches.getqId()); + + if (pModelTile == null || qModelTile == null) { + LOG.info("alignTiles: skipping pair {} in {}", canvasMatches.toKeyString(), this); + continue; + } + final List pointMatches = CanvasMatchResult.convertMatchesToPointMatchList(canvasMatches.getMatches()); + pModelTile.connect(qModelTile, pointMatches); + connectedPairCount++; + } + + LOG.info("alignTiles: start optimization after connecting {} pairs, {}", + connectedPairCount, this); // Optimize the tiles final TileConfiguration tc = new TileConfiguration(); @@ -382,6 +489,11 @@ private ResolvedTileSpecCollection intensityCorrectTiles( final ImageProcessorCache cache ) { + LOG.info("intensityCorrectTiles: entry with {} tiles and {} match pairs for {}", + tiles.getTileCount(), + tilePairs.size(), + this); + final Map> modelTiles = new HashMap<>(); // Initialize models for each tile spec @@ -424,8 +536,8 @@ private ResolvedTileSpecCollection intensityCorrectTiles( modelTileI.connect(modelTileJ, Collections.singletonList(pointMatch)); } else { - LOG.warn("intensityCorrectTiles: tiles {} and {} do not overlap after alignment", - tileIdI, tileIdJ); + LOG.warn("intensityCorrectTiles: tiles {} and {} do not overlap after alignment of {}", + tileSpecI.toTileBounds(), tileSpecJ.toTileBounds(), this); } } @@ -438,7 +550,7 @@ private ResolvedTileSpecCollection intensityCorrectTiles( final ErrorStatistic errorStatistic = new ErrorStatistic(1001); tc.optimizeSilently(errorStatistic, 0.5, 1000, 100); } catch (final NotEnoughDataPointsException | IllDefinedDataPointsException e) { - throw new RuntimeException("Failed to optimize intensity correction", e); + throw new RuntimeException("Failed to optimize intensity correction for " + this, e); } // Apply the intensity shift to the tiles diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/multisem/MfovPrealignTaskTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/multisem/MfovPrealignTaskTest.java index 6988cafae..b3b5849c4 100644 --- a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/multisem/MfovPrealignTaskTest.java +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/multisem/MfovPrealignTaskTest.java @@ -6,6 +6,7 @@ import java.util.Date; import java.util.List; +import org.janelia.alignment.match.MatchCollectionId; import org.janelia.alignment.multisem.LayerMFOV; import org.janelia.alignment.spec.stack.StackId; import org.janelia.alignment.spec.stack.StackMetaData; @@ -23,6 +24,37 @@ public class MfovPrealignTaskTest { public static void main(final String[] args) throws Exception { + final String baseDataUrl = "http://renderer-dev.int.janelia.org:8080/render-ws/v1"; + final String owner = "hess_wafers_60_61"; + final String project = "w61_serial_140_to_149"; + final StackId rawSfovStackId = new StackId(owner, project,"w61_s140_r00_gc_icc_test_two_tiles"); + final LayerMFOV z1LayerMfov = new LayerMFOV(1.0, "0023_m0014"); + + // final MatchCollectionId matchCollectionId = null; // set collection to null to derive matches during run + final MatchCollectionId matchCollectionId = new MatchCollectionId(owner, "w61_s140_r00_gc_test_z_1_and_2_match"); + + final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd_HHmmss"); + final String prealignedStackSuffix = "_pa_" + sdf.format(new Date()); + final StackId prealignedStackId = rawSfovStackId.withStackSuffix(prealignedStackSuffix); + + final RenderDataClient dataClient = new RenderDataClient(baseDataUrl, owner, project); + final StackMetaData rawStackMetaData = dataClient.getStackMetaData(rawSfovStackId.getStack()); + dataClient.setupDerivedStack(rawStackMetaData, prealignedStackId.getStack()); + + final MfovPrealignTask z1Task = + new MfovPrealignTask(baseDataUrl, + rawSfovStackId, + prealignedStackId, + z1LayerMfov, + matchCollectionId); + z1Task.run(); + + dataClient.setStackState(prealignedStackId.getStack(), StackMetaData.StackState.COMPLETE); + } + + @SuppressWarnings("unused") + public static void main_2025(final String[] args) throws Exception { + final String mfovRootDirectory = "/nrs/hess/data/hess_wafers_60_61/tiles_mfov"; if (! Files.exists(Paths.get(mfovRootDirectory))) { throw new RuntimeException("The mfovRootDirectory " + mfovRootDirectory + " does not exist " + @@ -49,7 +81,8 @@ public static void main(final String[] args) throws Exception { new MfovPrealignTask(baseDataUrl, rawSfovStackId, prealignedStackId, - z17LayerMfov); + z17LayerMfov, + null); z17Task.run(); final LayerMFOV z18LayerMfov = new LayerMFOV(18.0, "0399_m0035"); @@ -57,7 +90,8 @@ public static void main(final String[] args) throws Exception { new MfovPrealignTask(baseDataUrl, rawSfovStackId, prealignedStackId, - z18LayerMfov); + z18LayerMfov, + null); z18Task.run(); dataClient.setStackState(prealignedStackId.getStack(), StackMetaData.StackState.COMPLETE); @@ -101,5 +135,4 @@ public static void main(final String[] args) throws Exception { dataClient.setStackState(renderedMfovStackId.getStack(), StackMetaData.StackState.COMPLETE); } - } From fd054ed22e6b1728ddb54fffa34806ac4f15bcf6 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Mon, 10 Aug 2026 11:35:18 -0400 Subject: [PATCH 31/40] first draft of the MatchCollectionRenameClient --- .../MatchCollectionRenameParameters.java | 72 ++++++++ .../multisem/MatchCollectionRenameClient.java | 154 ++++++++++++++++++ .../pipeline/AlignmentPipelineParameters.java | 9 + .../pipeline/AlignmentPipelineStepId.java | 2 + 4 files changed, 237 insertions(+) create mode 100644 render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchCollectionRenameParameters.java create mode 100644 render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MatchCollectionRenameClient.java diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchCollectionRenameParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchCollectionRenameParameters.java new file mode 100644 index 000000000..15a66d28f --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchCollectionRenameParameters.java @@ -0,0 +1,72 @@ +package org.janelia.render.client.parameter; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import java.io.Serializable; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * Parameters for renaming match collections. + */ +@Parameters +public class MatchCollectionRenameParameters + implements Serializable { + + @Parameter( + names = "--matchCollectionOwner", + description = "Owner of the match collections to rename") + public String owner; + + @Parameter( + names = "--sourceNamePattern", + description = "Regular expression for the names of the collections to rename " + + "(e.g. '(w6.*_gc_icc)_match')") + public String sourceNamePattern; + + @Parameter( + names = "--targetNamePattern", + description = "Replacement for renamed collections that can reference capture groups " + + "from the sourceNamePattern (e.g. '$1_par_match' renames " + + "w61_s140_r00_gc_icc_match to w61_s140_r00_gc_icc_par_match)") + public String targetNamePattern; + + public MatchCollectionRenameParameters() { + } + + public void validate() + throws IllegalArgumentException { + + if ((owner == null) || (owner.trim().isEmpty())) { + throw new IllegalArgumentException("--matchCollectionOwner must be defined"); + } + + if ((sourceNamePattern == null) || (sourceNamePattern.trim().isEmpty())) { + throw new IllegalArgumentException("--sourceNamePattern must be defined"); + } + + if ((targetNamePattern == null) || (targetNamePattern.trim().isEmpty())) { + throw new IllegalArgumentException("--targetNamePattern must be defined"); + } + + buildSourceNamePattern(); // throws exception if the pattern is invalid + } + + public Pattern buildSourceNamePattern() + throws IllegalArgumentException { + try { + return Pattern.compile(sourceNamePattern); + } catch (final PatternSyntaxException e) { + throw new IllegalArgumentException("invalid --sourceNamePattern '" + sourceNamePattern + "'", e); + } + } + + @Override + public String toString() { + return "{owner='" + owner + '\'' + + ", sourceNamePattern='" + sourceNamePattern + '\'' + + ", targetNamePattern='" + targetNamePattern + '\'' + + '}'; + } +} diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MatchCollectionRenameClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MatchCollectionRenameClient.java new file mode 100644 index 000000000..85ce62dc6 --- /dev/null +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MatchCollectionRenameClient.java @@ -0,0 +1,154 @@ +package org.janelia.render.client.spark.multisem; + +import com.beust.jcommander.ParametersDelegate; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.apache.spark.api.java.JavaSparkContext; +import org.janelia.alignment.match.MatchCollectionId; +import org.janelia.render.client.ClientRunner; +import org.janelia.render.client.RenderDataClient; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MatchCollectionRenameParameters; +import org.janelia.render.client.parameter.MultiProjectParameters; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineParameters; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineStep; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineStepId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Client for renaming an owner's match collections (e.g. so that collection names stay derived from + * the names of the stacks that later pipeline steps use). + * + *

    Renames are quick web service calls, so nothing is distributed to spark workers here.

    + */ +public class MatchCollectionRenameClient + implements Serializable, AlignmentPipelineStep { + + public static class Parameters extends CommandLineParameters { + + @ParametersDelegate + public MultiProjectParameters multiProject = new MultiProjectParameters(); + + @ParametersDelegate + public MatchCollectionRenameParameters matchCollectionRename = new MatchCollectionRenameParameters(); + } + + public static void main(final String[] args) { + final ClientRunner clientRunner = new ClientRunner(args) { + @Override + public void runClient(final String[] args) throws Exception { + final Parameters parameters = new Parameters(); + parameters.parse(args); + parameters.matchCollectionRename.validate(); + + LOG.info("runClient: entry, parameters={}", parameters); + + // NOTE: no spark context is needed here because all renames are run on the driver + final MatchCollectionRenameClient client = new MatchCollectionRenameClient(); + client.renameMatchCollections(parameters.multiProject.getBaseDataUrl(), + parameters.matchCollectionRename); + } + }; + clientRunner.run(); + } + + public MatchCollectionRenameClient() { + } + + /** Validates the specified pipeline parameters are sufficient. */ + @Override + public void validatePipelineParameters(final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException { + final MatchCollectionRenameParameters matchCollectionRename = pipelineParameters.getMatchCollectionRename(); + AlignmentPipelineParameters.validateRequiredElementExists("matchCollectionRename", + matchCollectionRename); + matchCollectionRename.validate(); + } + + /** Run the client as part of an alignment pipeline. */ + @Override + public void runPipelineStep(final JavaSparkContext sparkContext, + final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException, IOException { + + final MultiProjectParameters multiProject = pipelineParameters.getMultiProject(null); + + renameMatchCollections(multiProject.getBaseDataUrl(), + pipelineParameters.getMatchCollectionRename()); + } + + @Override + public AlignmentPipelineStepId getDefaultStepId() { + return AlignmentPipelineStepId.RENAME_MATCH_COLLECTIONS; + } + + public void renameMatchCollections(final String baseDataUrl, + final MatchCollectionRenameParameters matchCollectionRename) + throws IllegalArgumentException, IOException { + + LOG.info("renameMatchCollections: entry, matchCollectionRename={}", matchCollectionRename); + + final String owner = matchCollectionRename.owner; + final RenderDataClient ownerDataClient = new RenderDataClient(baseDataUrl, owner, "not_used"); + + final List existingNames = ownerDataClient.getOwnerMatchCollections().stream() + .map(mcmd -> mcmd.getCollectionId().getName()) + .sorted() + .collect(Collectors.toList()); + + final Pattern sourcePattern = matchCollectionRename.buildSourceNamePattern(); + + // sourceNames and targetNames are parallel lists, so targetNames.get(i) is the new name for sourceNames.get(i) + final List sourceNames = new ArrayList<>(); + final List targetNames = new ArrayList<>(); + for (final String existingName : existingNames) { + final Matcher matcher = sourcePattern.matcher(existingName); + if (matcher.matches()) { + sourceNames.add(existingName); + targetNames.add(matcher.replaceFirst(matchCollectionRename.targetNamePattern)); + } + } + + // check all renames before doing any of them so that partial renames are less likely + final Set existingNameSet = new HashSet<>(existingNames); + final Set distinctTargetNames = new HashSet<>(); + for (int i = 0; i < targetNames.size(); i++) { + final String targetName = targetNames.get(i); + if (existingNameSet.contains(targetName)) { + throw new IllegalArgumentException("cannot rename " + sourceNames.get(i) + " to " + targetName + + " because a collection with that name already exists for owner " + + owner); + } + if (! distinctTargetNames.add(targetName)) { + throw new IllegalArgumentException("cannot rename more than one collection to " + targetName + + " for owner " + owner); + } + } + + if (targetNames.isEmpty()) { + LOG.warn("renameMatchCollections: none of the {} collection(s) for owner {} match the sourceNamePattern '{}'", + existingNames.size(), owner, matchCollectionRename.sourceNamePattern); + } + + for (int i = 0; i < targetNames.size(); i++) { + final String sourceName = sourceNames.get(i); + final MatchCollectionId targetCollectionId = new MatchCollectionId(owner, targetNames.get(i)); + final RenderDataClient matchDataClient = new RenderDataClient(baseDataUrl, owner, sourceName); + matchDataClient.renameMatchCollection(targetCollectionId); + } + + LOG.info("renameMatchCollections: exit, renamed {} collection(s)", targetNames.size()); + } + + private static final Logger LOG = LoggerFactory.getLogger(MatchCollectionRenameClient.class); +} diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java index 0795172b2..c899250af 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java @@ -22,6 +22,7 @@ import org.janelia.render.client.parameter.MFOVAsTileParameters; import org.janelia.render.client.parameter.MFOVMontageMatchPatchParameters; import org.janelia.render.client.parameter.MaskHackParameters; +import org.janelia.render.client.parameter.MatchCollectionRenameParameters; import org.janelia.render.client.parameter.MatchCopyParameters; import org.janelia.render.client.parameter.MipmapParameters; import org.janelia.render.client.parameter.MultiProjectParameters; @@ -51,6 +52,7 @@ public class AlignmentPipelineParameters private final UnconnectedCrossMFOVParameters unconnectedCrossMfov; private final TileClusterParameters tileCluster; private final MatchCopyParameters matchCopy; + private final MatchCollectionRenameParameters matchCollectionRename; private final BeamCorrectionParameters beamCorrection; private final CreepCorrectionParameters creepCorrection; private final AffineBlockSolverSetup affineBlockSolverSetup; @@ -82,6 +84,7 @@ public AlignmentPipelineParameters() { null, null, null, + null, null); } @@ -94,6 +97,7 @@ public AlignmentPipelineParameters(final MultiProjectParameters multiProject, final UnconnectedCrossMFOVParameters unconnectedCrossMfov, final TileClusterParameters tileCluster, final MatchCopyParameters matchCopy, + final MatchCollectionRenameParameters matchCollectionRename, final BeamCorrectionParameters beamCorrection, final CreepCorrectionParameters creepCorrection, final AffineBlockSolverSetup affineBlockSolverSetup, @@ -113,6 +117,7 @@ public AlignmentPipelineParameters(final MultiProjectParameters multiProject, this.unconnectedCrossMfov = unconnectedCrossMfov; this.tileCluster = tileCluster; this.matchCopy = matchCopy; + this.matchCollectionRename = matchCollectionRename; this.beamCorrection = beamCorrection; this.creepCorrection = creepCorrection; this.affineBlockSolverSetup = affineBlockSolverSetup; @@ -175,6 +180,10 @@ public MatchCopyParameters getMatchCopy() { return matchCopy; } + public MatchCollectionRenameParameters getMatchCollectionRename() { + return matchCollectionRename; + } + public BeamCorrectionParameters getBeamCorrection() { return beamCorrection; } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java index b09ef2915..9dba4fb2e 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java @@ -13,6 +13,7 @@ import org.janelia.render.client.spark.multisem.LayerAsTileClient; import org.janelia.render.client.spark.multisem.MFOVAsTileClient; import org.janelia.render.client.spark.multisem.MFOVMontageMatchPatchClient; +import org.janelia.render.client.spark.multisem.MatchCollectionRenameClient; import org.janelia.render.client.spark.multisem.UnconnectedCrossMFOVClient; import org.janelia.render.client.spark.newsolver.DistributedAffineBlockSolverClient; import org.janelia.render.client.spark.newsolver.DistributedIntensityCorrectionBlockSolverClient; @@ -41,6 +42,7 @@ public enum AlignmentPipelineStepId { RENDER_SCAPE_IMAGES(ScapeClient::new), RENDER_TILES(RenderTilesClient::new), MFOV_AS_TILE(MFOVAsTileClient::new), + RENAME_MATCH_COLLECTIONS(MatchCollectionRenameClient::new), LAYER_AS_TILE(LayerAsTileClient::new); private final Supplier stepClientSupplier; From 805defa8f6be1c785e9baa29c505dfc11b6ff95f Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Mon, 10 Aug 2026 13:56:53 -0400 Subject: [PATCH 32/40] rename owner as matchCollectionOwner to keep parameter name and attribute name consistent --- .../client/parameter/MatchCollectionRenameParameters.java | 6 +++--- .../client/spark/multisem/MatchCollectionRenameClient.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchCollectionRenameParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchCollectionRenameParameters.java index 15a66d28f..b0ca50b39 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchCollectionRenameParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchCollectionRenameParameters.java @@ -17,7 +17,7 @@ public class MatchCollectionRenameParameters @Parameter( names = "--matchCollectionOwner", description = "Owner of the match collections to rename") - public String owner; + public String matchCollectionOwner; @Parameter( names = "--sourceNamePattern", @@ -38,7 +38,7 @@ public MatchCollectionRenameParameters() { public void validate() throws IllegalArgumentException { - if ((owner == null) || (owner.trim().isEmpty())) { + if ((matchCollectionOwner == null) || (matchCollectionOwner.trim().isEmpty())) { throw new IllegalArgumentException("--matchCollectionOwner must be defined"); } @@ -64,7 +64,7 @@ public Pattern buildSourceNamePattern() @Override public String toString() { - return "{owner='" + owner + '\'' + + return "{owner='" + matchCollectionOwner + '\'' + ", sourceNamePattern='" + sourceNamePattern + '\'' + ", targetNamePattern='" + targetNamePattern + '\'' + '}'; diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MatchCollectionRenameClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MatchCollectionRenameClient.java index 85ce62dc6..e760595e0 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MatchCollectionRenameClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MatchCollectionRenameClient.java @@ -98,7 +98,7 @@ public void renameMatchCollections(final String baseDataUrl, LOG.info("renameMatchCollections: entry, matchCollectionRename={}", matchCollectionRename); - final String owner = matchCollectionRename.owner; + final String owner = matchCollectionRename.matchCollectionOwner; final RenderDataClient ownerDataClient = new RenderDataClient(baseDataUrl, owner, "not_used"); final List existingNames = ownerDataClient.getOwnerMatchCollections().stream() From 79af5240a35ae08c2dfdccede175295738b0adbf Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Mon, 10 Aug 2026 14:00:45 -0400 Subject: [PATCH 33/40] Treat resinMfovStartPositionMatchWeight as unspecified if it is negative. This makes it a little clearer that you are excluding cross MFOV match pair patching when you specify a negative weight (rather than leaving the weight unspecified). --- .../multisem/UnconnectedMontageMFOVClient.java | 15 ++++++++++++--- .../MFOVMontageMatchPatchParameters.java | 12 +++++++++++- .../multisem/MFOVMontageMatchPatchClient.java | 2 +- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/UnconnectedMontageMFOVClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/UnconnectedMontageMFOVClient.java index fa10312b2..d9bd115c8 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/UnconnectedMontageMFOVClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/UnconnectedMontageMFOVClient.java @@ -57,10 +57,19 @@ public static class Parameters @Parameter( names = "--startPositionMatchWeight", description = "Weight (e.g. 0.001) for matches derived from SFOV start positions. " + - "Specify to patch all unconnected pairs with positions based upon SFOV stage locations. " + - "Omit to skip start position derivation.") + "Specify a positive value to patch all unconnected pairs with positions based upon " + + "SFOV stage locations. " + + "Omit (or specify a non-positive value) to skip start position derivation.") public Double startPositionMatchWeight; + /** + * @return the start position match weight or null if derivation should be skipped + * (non-positive weights are used to exclude cross MFOV pairs from patching). + */ + public Double getStartPositionMatchWeight() { + return ((startPositionMatchWeight != null) && (startPositionMatchWeight > 0.0)) ? + startPositionMatchWeight : null; + } } /** Label for tiles in MFOVs with isolated edges. */ @@ -105,7 +114,7 @@ public void findIsolatedMFOVs() parameters.multiProject.deriveMatchCollectionNamesFromProject, renderDataClient, parameters.addIsolatedEdgeLabel, - parameters.startPositionMatchWeight); + parameters.getStartPositionMatchWeight()); } } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVMontageMatchPatchParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVMontageMatchPatchParameters.java index 763550342..e99755edd 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVMontageMatchPatchParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVMontageMatchPatchParameters.java @@ -213,8 +213,18 @@ public String getTrimStackName(final String sourceStackName) { return trimMfovsWithNoConnectedTiles ? sourceStackName + "_trim" : null; } + /** + * @return the weight for matches derived from SFOV start positions + * or null if start position derivation should be skipped + * (non-positive weights are used to exclude cross MFOV pairs from patching). + */ + public Double getResinMfovStartPositionMatchWeight() { + return ((resinMfovStartPositionMatchWeight != null) && (resinMfovStartPositionMatchWeight > 0.0)) ? + resinMfovStartPositionMatchWeight : null; + } + public boolean isIsolatedMfovPatchingNeeded() { - return addIsolatedEdgeLabel || resinMfovStartPositionMatchWeight != null; + return addIsolatedEdgeLabel || (getResinMfovStartPositionMatchWeight() != null); } public static MFOVMontageMatchPatchParameters fromJson(final Reader json) { diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVMontageMatchPatchClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVMontageMatchPatchClient.java index 8587e55d4..aeb04412b 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVMontageMatchPatchClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVMontageMatchPatchClient.java @@ -195,7 +195,7 @@ private void patchIsolatedMFOVs(final JavaSparkContext sparkContext, false, renderDataClient, patchParameters.addIsolatedEdgeLabel, - patchParameters.resinMfovStartPositionMatchWeight); + patchParameters.getResinMfovStartPositionMatchWeight()); return null; }; From b99c798a9d78c1cae1cb6e331c14dab1e467b52d Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Mon, 10 Aug 2026 20:10:23 -0400 Subject: [PATCH 34/40] fix expectedTileClusterCount for MFOV montage match patching when isolated MFOV patching is not done --- .../multisem/MFOVMontageMatchPatchClient.java | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVMontageMatchPatchClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVMontageMatchPatchClient.java index aeb04412b..bcd741f77 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVMontageMatchPatchClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVMontageMatchPatchClient.java @@ -104,6 +104,7 @@ public void createContextAndRun(final Parameters clientParameters) throws IOExce if (patchParameters.checkLayerConnectedClusters) { checkLayerConnectedClusters(sparkContext, clientParameters.multiProject, + patchParameters, tileClusterParameters); } @@ -135,7 +136,10 @@ public void runPipelineStep(final JavaSparkContext sparkContext, patchMFOVs(sparkContext, multiProjectParameters, patchParameters); if (patchParameters.checkLayerConnectedClusters) { - checkLayerConnectedClusters(sparkContext, multiProjectParameters, pipelineParameters.getTileCluster()); + checkLayerConnectedClusters(sparkContext, + multiProjectParameters, + patchParameters, + pipelineParameters.getTileCluster()); } } @@ -319,6 +323,7 @@ private void completeTrimStacks(final MultiProjectParameters multiProject) private void checkLayerConnectedClusters(final JavaSparkContext sparkContext, final MultiProjectParameters multiProjectParameters, + final MFOVMontageMatchPatchParameters patchParameters, final TileClusterParameters tileClusterParameters) throws IOException { @@ -331,7 +336,6 @@ private void checkLayerConnectedClusters(final JavaSparkContext sparkContext, final List connectedTileClusterSummaryForStacks = clusterCountClient.findConnectedClusters(sparkContext, clientParameters); - final List stackErrors = new ArrayList<>(); for (final ConnectedTileClusterSummaryForStack summary : connectedTileClusterSummaryForStacks) { @@ -339,7 +343,25 @@ private void checkLayerConnectedClusters(final JavaSparkContext sparkContext, final RenderDataClient renderDataClient = multiProjectParameters.getDataClient().buildClient(stackId.getOwner(), stackId.getProject()); final List zValues = renderDataClient.getStackZValues(stackId.getStack()); - final String countErrorString = summary.buildCountErrorString(zValues.size(), + + int expectedTileClusterCount = zValues.size(); + if (! patchParameters.isIsolatedMfovPatchingNeeded()) { + // resin MFOV patching derives the only cross MFOV matches (isolated edge labelling derives none), + // so without it the expected cluster count is the total number of layer MFOVs + expectedTileClusterCount = 0; + for (final Double z : zValues) { + final List sortedMFOVNames = + MultiProjectParameters.getSortedMFOVNamesForOneLayer(renderDataClient, + stackId.getStack(), + z); + expectedTileClusterCount += sortedMFOVNames.size(); + } + } + + LOG.info("checkLayerConnectedClusters: expectedTileClusterCount is {} for {}", + expectedTileClusterCount, stackId.toDevString()); + + final String countErrorString = summary.buildCountErrorString(expectedTileClusterCount, 0, 0); if (! countErrorString.isEmpty()) { From 46149f5976d9f7814f108b5519f6181841a9ff2c Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Wed, 12 Aug 2026 19:51:18 -0400 Subject: [PATCH 35/40] remove unneeded writeable directory checks for Google Cloud storage that wasted compute time (e.g. 15 minutes of a 90-minute run) --- .../render/client/tile/RenderTilesClient.java | 52 ++++++++----------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/tile/RenderTilesClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/tile/RenderTilesClient.java index 6e5327e1f..f63d70b06 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/tile/RenderTilesClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/tile/RenderTilesClient.java @@ -304,33 +304,33 @@ public void setupHackStackAsNeeded() public void setupStorageDirectories() throws IOException { - for (final double z : renderDataClient.getStackZValues(stack)) { + LOG.info("setupStorageDirectories: entry"); - final List relativePathSegments = getImageParentPathSegments(z); - final URI parentUri = storageBackend.resolvePath(relativePathSegments); + if (storageBackend instanceof CloudStorage) { - LOG.info("setupStorageDirectories: ensuring writable directory (or bucket 'prefix') for z {} at {}", - z, parentUri); - try { - storageBackend.ensureWritableDirectory(parentUri); - - final String scheme = storageBackend.root.getScheme(); - if ((scheme != null) && scheme.startsWith("gs:")) { - // Supposedly, ~1000 object mutations per second per bucket per project are allowed by Google. - // We have seen "exceeded the rate limit for object mutation operations" exceptions even after - // limiting the rate to 20 per second. - // Limiting the rate to 10 per second to try to avoid this. - Thread.sleep(100); - } + LOG.info("setupStorageDirectories: skipping since there is no need to set anything up for CloudStorage"); + + } else { + + for (final double z : renderDataClient.getStackZValues(stack)) { + + final List relativePathSegments = getImageParentPathSegments(z); + final URI parentUri = storageBackend.resolvePath(relativePathSegments); + + LOG.info("setupStorageDirectories: ensuring writable directory for z {} at {}", z, parentUri); - } catch (final Throwable t) { - LOG.warn("setupStorageDirectories: caught exception and will retry setup in 5 seconds", t); try { - Thread.sleep(5000); - } catch (final InterruptedException ie) { - LOG.warn("setupStorageDirectories: caught exception while sleeping and will retry setup now", ie); + storageBackend.ensureWritableDirectory(parentUri); + } catch (final Throwable t) { + LOG.warn("setupStorageDirectories: caught exception and will retry setup in 5 seconds", t); + try { + Thread.sleep(5000); + } catch (final InterruptedException ie) { + LOG.warn("setupStorageDirectories: caught exception while sleeping and will retry setup now", ie); + } + storageBackend.ensureWritableDirectory(parentUri); } - storageBackend.ensureWritableDirectory(parentUri); + } } @@ -656,13 +656,7 @@ private static class CloudStorage extends StorageBackend { @Override void ensureWritableDirectory(final URI uri) { - if (!keyValueAccess.exists(uri.getPath())) { - try { - keyValueAccess.createDirectories(uri.getPath()); - } catch (final IOException e) { - throw new RuntimeException("Could not create directory " + uri, e); - } - } + // no-op for cloud storage } @Override From 24118a69eb0dc08cbf69a88a97e9101b698dedf0 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Mon, 17 Aug 2026 15:24:28 -0400 Subject: [PATCH 36/40] increase stack serial number range --- render-ws-with-mongo-db/db-dump-google-collections.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/render-ws-with-mongo-db/db-dump-google-collections.sh b/render-ws-with-mongo-db/db-dump-google-collections.sh index 69c61e718..3d68f96a7 100755 --- a/render-ws-with-mongo-db/db-dump-google-collections.sh +++ b/render-ws-with-mongo-db/db-dump-google-collections.sh @@ -132,7 +132,7 @@ else echo " Select project:" PROJECTS=() - for i in $(seq 0 10 150); do + for i in $(seq 0 10 400); do PROJECTS+=("$(printf "w61_serial_%03d_to_%03d" "$i" "$((i+9))")") done select PROJECT in "${PROJECTS[@]}"; do From 716b1730befef19555b5c6bf085593447ff91413 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Mon, 17 Aug 2026 15:26:11 -0400 Subject: [PATCH 37/40] add load-rough-align-data.sh script --- .../other/load-rough-align-data.sh | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 render-ws-with-mongo-db/other/load-rough-align-data.sh diff --git a/render-ws-with-mongo-db/other/load-rough-align-data.sh b/render-ws-with-mongo-db/other/load-rough-align-data.sh new file mode 100644 index 000000000..f4067d8d1 --- /dev/null +++ b/render-ws-with-mongo-db/other/load-rough-align-data.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# Batch identifier appended to each slab group name (edit this for each round of runs). +SLAB_GROUP_SUFFIX="20260811b" + +VM_IPS=(10.150.0.2 10.150.0.3 10.150.0.4 10.150.0.5 10.150.0.6 10.150.0.7) + +printf "\nWhich VM do you want to use?\n\n" +select VM_IP in "${VM_IPS[@]}"; do + if [ -n "${VM_IP}" ]; then + break + else + echo "Invalid selection, try again." + fi +done + +printf "\nWhich wafer do you want to use?\n\n" +select WAFER in 60 61; do + if [ -n "${WAFER}" ]; then + break + else + echo "Invalid selection, try again." + fi +done + +echo +read -rp "Enter the first serial number (a multiple of 5 between 0 and 410): " FIRST_SERIAL_NUMBER + +if [[ ! ${FIRST_SERIAL_NUMBER} =~ ^[0-9]+$ ]]; then + printf "\nExiting, '%s' is not a number\n\n" "${FIRST_SERIAL_NUMBER}" + exit 1 +fi + +# force base 10 so that zero padded values (e.g. 070) are not treated as octal +FIRST_SERIAL_NUMBER=$(( 10#${FIRST_SERIAL_NUMBER} )) + +if (( FIRST_SERIAL_NUMBER > 410 )) || (( FIRST_SERIAL_NUMBER % 5 != 0 )); then + printf "\nExiting, %d is not a multiple of 5 between 0 and 410\n\n" "${FIRST_SERIAL_NUMBER}" + exit 1 +fi + +LAST_SERIAL_NUMBER=$(( FIRST_SERIAL_NUMBER + 4 )) + +# even serial numbers are the first half of a project's slabs, odd ones are the second half +if (( FIRST_SERIAL_NUMBER % 2 == 0 )); then + KEEP_OR_REMOVE="[k]ept" + FIRST_PROJECT_NUMBER=${FIRST_SERIAL_NUMBER} +else + KEEP_OR_REMOVE="[r]emoved" + FIRST_PROJECT_NUMBER=$(( FIRST_SERIAL_NUMBER - 5 )) +fi + +LAST_PROJECT_NUMBER=$(( FIRST_PROJECT_NUMBER + 9 )) + +FIRST_SERIAL=$(printf "%03d" "${FIRST_SERIAL_NUMBER}") +LAST_SERIAL=$(printf "%03d" "${LAST_SERIAL_NUMBER}") +FIRST_PROJECT=$(printf "%03d" "${FIRST_PROJECT_NUMBER}") +LAST_PROJECT=$(printf "%03d" "${LAST_PROJECT_NUMBER}") + +SLAB_GROUP="s${FIRST_SERIAL}_to_s${LAST_SERIAL}_${SLAB_GROUP_SUFFIX}" +BATCH_NAME="rough-w${WAFER}-s${FIRST_SERIAL}-to-s${LAST_SERIAL}" +PROJECT_GROUP="w${WAFER}_serial_${FIRST_PROJECT}_to_${LAST_PROJECT}" + +echo " +Set up for slab group ${SLAB_GROUP} from project group ${PROJECT_GROUP}: + + On ${VM_IP}, run: + + ./db-restore-collections.sh --pattern 'janelia/00_gc/.*s${FIRST_PROJECT}' + + ./other/remove-stacks.sh + + you want stacks to be ${KEEP_OR_REMOVE} + then enter ' 1 2 3 4 5 6 7 8 9 10 ' + + + On launch box, run: + + ./02_run_pipeline.sh ${VM_IP} 00_rough_align/pipe.00.w${WAFER}.icc-match-mat.json 120 4 premium 120 ${BATCH_NAME} + + + After the run completes (typically 8 to 12 hours), on ${VM_IP}, run: + + ./list-stacks.sh + + ./list-match-collections.sh + + ./db-dump-google-collections.sh + + Select database: render + Select stage: 00_par + Select project: ${PROJECT_GROUP} + Enter slab-group: ${SLAB_GROUP} + + Enter collection pattern regex: .* + + Should dump collections to: + /mnt/disks/mongodb_dump_fs/dump/google/00_par/${PROJECT_GROUP}/${SLAB_GROUP}/render + + ./db-dump-google-collections.sh + + Select database: match + Select stage: 00_par + Select project: ${PROJECT_GROUP} + Enter slab-group: ${SLAB_GROUP} + + Enter collection pattern regex: .* + + Should dump collections to: + /mnt/disks/mongodb_dump_fs/dump/google/00_par/${PROJECT_GROUP}/${SLAB_GROUP}/match +" From a0e142e318eb1428601d4f0a619786227a16803e Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Mon, 17 Aug 2026 15:28:24 -0400 Subject: [PATCH 38/40] debug w61 mfov-as-tile match connection issues --- .../render/client/ClusterCountClientTest.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/ClusterCountClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/ClusterCountClientTest.java index 98e4608df..1de17219e 100644 --- a/render-ws-java-client/src/test/java/org/janelia/render/client/ClusterCountClientTest.java +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/ClusterCountClientTest.java @@ -41,8 +41,8 @@ public static void main(final String[] args) // "--maxSmallClusterSizeToSave", "100" }; - ClusterCountClient.main(effectiveArgs); -// findWafer61Clusters(); +// ClusterCountClient.main(effectiveArgs); + findWafer61Clusters(); } @@ -53,9 +53,9 @@ public static void findWafer61Clusters() throws Exception { final String baseDataUrl = "http://em-services-1:8080/render-ws/v1"; final String owner = "hess_wafers_60_61"; - final String project = "w61_serial_100_to_109"; - final String stack = "w61_s108_r00_gc_par"; - final int lastZ = 96; + final String project = "w61_serial_170_to_179"; + final String stack = "w61_s171_r01_gc_icc_pa_mat_render"; + final int lastZ = 82; final String mcName = stack + "_match"; final StackId stackId = new StackId(owner, project, stack); @@ -71,10 +71,9 @@ public static void findWafer61Clusters() throws Exception { jcccp.multiProject = MultiProjectParameters.singleStackInstance(baseDataUrl, stackId); jcccp.tileCluster = new TileClusterParameters(); - final int zCount = 96; jcccp.tileCluster.maxSmallClusterSize = 0; jcccp.tileCluster.includeMatchesOutsideGroup = true; - jcccp.tileCluster.maxLayersPerBatch = zCount + 1; + jcccp.tileCluster.maxLayersPerBatch = lastZ + 1; jcccp.tileCluster.maxOverlapLayers = 6; final RenderDataClient dataClient = new RenderDataClient(baseDataUrl, owner, project); From 163b3bebdaffa3cfad41808d03307ddf8fca3df6 Mon Sep 17 00:00:00 2001 From: "Eric T. Trautman" Date: Mon, 17 Aug 2026 15:42:48 -0400 Subject: [PATCH 39/40] fix exception message logic Co-authored-by: Michael Innerberger --- .../spec/ResolvedTileSpecsWithMatchPairs.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java b/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java index 9a621f3e6..49e95cd34 100644 --- a/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java +++ b/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java @@ -201,13 +201,13 @@ public void changeMatchesToWorldCoordinates() } else { final StringBuilder sb = new StringBuilder(); - if (pTileSpec == null) { - if (qTileSpec == null) { - sb.append("s ").append(pair.getpId()).append(" and ").append(pair.getqId()); + if (qTileSpec == null) { + if (pTileSpec == null) { + sb.append("s ").append(pair.getpId()).append(" and "); } - sb.append(" ").append(pair.getpId()); - } else { sb.append(" ").append(pair.getqId()); + } else { + sb.append(" ").append(pair.getpId()); } throw new IllegalStateException("missing tile" + sb + " for match pair " + pair.toKeyString()); From 13f5d05eef0bb9e1ee0a0e423f9ad9ccb88e7db8 Mon Sep 17 00:00:00 2001 From: Eric Trautman Date: Mon, 17 Aug 2026 15:59:38 -0400 Subject: [PATCH 40/40] add comment to explain coding choice --- .../main/java/org/janelia/alignment/spec/TileBoundsRTree.java | 1 + 1 file changed, 1 insertion(+) diff --git a/render-app/src/main/java/org/janelia/alignment/spec/TileBoundsRTree.java b/render-app/src/main/java/org/janelia/alignment/spec/TileBoundsRTree.java index cbee6d0fd..1b5f8b6ef 100644 --- a/render-app/src/main/java/org/janelia/alignment/spec/TileBoundsRTree.java +++ b/render-app/src/main/java/org/janelia/alignment/spec/TileBoundsRTree.java @@ -336,6 +336,7 @@ public static Set getDistinctPairs(final TileBounds fromTil isNeighborCenterInRange(fromMinX, fromMaxX, toTile.getMinX(), toTile.getMaxX()) || isNeighborCenterInRange(fromMinY, fromMaxY, toTile.getMinY(), toTile.getMaxY())) { + // note: only execute multi-SEM string extraction and comparison code if flag indicates it is needed if (excludeSameMfovNeighbors && (pMfov.equals(MultiSemUtilities.getMagcMfovForTileId(qTileId)))) { continue; } else if (excludeDifferentMfovNeighbors && (! pMfov.equals(MultiSemUtilities.getMagcMfovForTileId(qTileId)))) {