diff --git a/src/main/java/net/preibisch/mvrecon/process/interestpointregistration/pairwise/MatcherPairwiseTools.java b/src/main/java/net/preibisch/mvrecon/process/interestpointregistration/pairwise/MatcherPairwiseTools.java
index f6a49265..20999692 100644
--- a/src/main/java/net/preibisch/mvrecon/process/interestpointregistration/pairwise/MatcherPairwiseTools.java
+++ b/src/main/java/net/preibisch/mvrecon/process/interestpointregistration/pairwise/MatcherPairwiseTools.java
@@ -291,12 +291,24 @@ public static < V, I extends InterestPoint > List< Pair< Pair< V, V >, PairwiseR
 			final Map<String, ? extends List<I>> mapA = interestpoints.get( pair.getA() );
 			final Map<String, ? extends List<I>> mapB = interestpoints.get( pair.getB() );
 
+			final HashMap<String, String > compared = new HashMap<>();
+
 			for ( final String labelA : mapA.keySet() )
 				for ( final String labelB : mapB.keySet() )
 				{
 					if ( !matchAcrossLabels && !labelA.equals( labelB ) )
 						continue;
 
+					if ( compared.containsKey( labelA ) && compared.get( labelA ).equals( labelB ) )
+						continue;
+
+					// remember what we already compared
+					compared.put( labelA, labelB );
+
+					// for matchAcross also the inverse (A>B means we also did B>A)
+					if ( matchAcrossLabels && !labelA.equals( labelB ) )
+						compared.put( labelB, labelA );
+
 					final List< I > listA, listB;
 
 					if ( matcher.requiresInterestPointDuplication() )