Skip to content

Commit

Permalink
wip/fix: toward avoiding list and exists
Browse files Browse the repository at this point in the history 
* cache logic updated
* tests updated
  • Loading branch information
@bogovicj
bogovicj committed Dec 3, 2024
1 parent cc42161 commit ddffae4
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 11 deletions.
32 changes: 29 additions & 3 deletions src/main/java/org/janelia/saalfeldlab/n5/zarr/cache/ZarrJsonCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache

N5CacheInfo cacheInfo = getCacheInfo(normalPathKey);
if (cacheInfo == null ){
addNewCacheInfo(normalPathKey, normalCacheKey, uncachedAttributes );
addNewCacheInfo(normalPathKey, normalCacheKey, uncachedAttributes);
return;
}

Expand Down Expand Up @@ -81,7 +81,7 @@ public boolean isDataset(final String normalPathKey, final String normalCacheKey

N5CacheInfo cacheInfo = getCacheInfo(normalPathKey);
if (cacheInfo == null) {
addNewCacheInfo(normalPathKey, normalCacheKey, null);
cacheGroupAndDataset(normalPathKey);
cacheInfo = getCacheInfo(normalPathKey);
}
else if (cacheInfo == emptyCacheInfo || cacheInfo.isGroup())
Expand All @@ -100,7 +100,7 @@ public boolean isGroup(final String normalPathKey, final String normalCacheKey)

N5CacheInfo cacheInfo = getCacheInfo(normalPathKey);
if (cacheInfo == null) {
addNewCacheInfo(normalPathKey, normalCacheKey, null);
cacheGroupAndDataset(normalPathKey);
cacheInfo = getCacheInfo(normalPathKey);
}
else if (cacheInfo == emptyCacheInfo || cacheInfo.isDataset())
Expand All @@ -114,4 +114,30 @@ else if (!cacheInfo.containsKey(ZarrKeyValueReader.ZGROUP_FILE)) {
return cacheInfo.isGroup();
}

public N5CacheInfo cacheGroupAndDataset(final String normalPathKey) {

final JsonElement zgroup = container.getAttributesFromContainer(normalPathKey, ZarrKeyValueReader.ZGROUP_FILE);
final boolean isGroup = container.isGroupFromAttributes(normalPathKey, zgroup);

final JsonElement zarray = container.getAttributesFromContainer(normalPathKey, ZarrKeyValueReader.ZARRAY_FILE);
final boolean isDataset = container.isGroupFromAttributes(normalPathKey, zarray);

if( isGroup || isDataset ) {

final N5CacheInfo cacheInfo = newCacheInfo();

updateCacheIsGroup(cacheInfo, isGroup);
updateCacheAttributes(cacheInfo, ZarrKeyValueReader.ZGROUP_FILE, zgroup);

updateCacheIsDataset(cacheInfo, isDataset);
updateCacheAttributes(cacheInfo, ZarrKeyValueReader.ZARRAY_FILE, zarray);

updateCache(normalPathKey, cacheInfo);
return cacheInfo;
}

updateCache(normalPathKey, emptyCacheInfo);
return emptyCacheInfo;
}

}
27 changes: 19 additions & 8 deletions src/test/java/org/janelia/saalfeldlab/n5/zarr/ZarrCachedFSTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,16 +168,17 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {
int expectedExistCount = 0;
final int expectedGroupCount = 0;
final int expectedDatasetCount = 0;
int expectedAttributeCount = 0;
int expectedAttributeCount = 0; // isGroup and isDataset are called when creating the reader
int expectedListCount = 0;

boolean exists = n5.exists(groupA);
boolean groupExists = n5.groupExists(groupA);
boolean datasetExists = n5.datasetExists(groupA);
expectedAttributeCount+=2; // attributes (zarray and zgroup) are called by groupExists and datasetExists
assertFalse(exists); // group does not exist
assertFalse(groupExists); // group does not exist
assertFalse(datasetExists); // dataset does not exist
assertEquals(++expectedExistCount, n5.getExistCallCount());
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedGroupCount, n5.getGroupCallCount());
assertEquals(expectedDatasetCount, n5.getDatasetCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
Expand All @@ -189,10 +190,11 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {
exists = n5.exists(groupB);
groupExists = n5.groupExists(groupB);
datasetExists = n5.datasetExists(groupB);
expectedAttributeCount+=2; // attributes (zarray and zgroup) are called by groupExists and datasetExists
assertFalse(exists); // group now exists
assertFalse(groupExists); // group now exists
assertFalse(datasetExists); // dataset does not exist
assertEquals(++expectedExistCount, n5.getExistCallCount());
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedGroupCount, n5.getGroupCallCount());
assertEquals(expectedDatasetCount, n5.getDatasetCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
Expand All @@ -212,7 +214,8 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {
// should not check existence when creating a group
n5.createGroup(cachedGroup);
n5.createGroup(cachedGroup); // be annoying
assertEquals(++expectedExistCount, n5.getExistCallCount());
expectedAttributeCount+=2; // createGroup calls isGroup and isDataset
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedGroupCount, n5.getGroupCallCount());
assertEquals(expectedDatasetCount, n5.getDatasetCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
Expand Down Expand Up @@ -271,7 +274,8 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {
*/
final String nonExistentGroup = "doesNotExist";
n5.exists(nonExistentGroup);
assertEquals(++expectedExistCount, n5.getExistCallCount());
expectedAttributeCount+=2; // exists calls isGroup and isDataset
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedGroupCount, n5.getGroupCallCount());
assertEquals(expectedDatasetCount, n5.getDatasetCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
Expand Down Expand Up @@ -310,10 +314,11 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {
final String abc = "a/b/c";
// create "a/b/c"
n5.createGroup(abc);
expectedAttributeCount+=2; // createGroup calls isGroup and isDataset
assertTrue(n5.exists(abc));
assertTrue(n5.groupExists(abc));
assertFalse(n5.datasetExists(abc));
assertEquals(++expectedExistCount, n5.getExistCallCount());
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedGroupCount, n5.getGroupCallCount());
assertEquals(expectedDatasetCount, n5.getDatasetCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
Expand Down Expand Up @@ -362,12 +367,18 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {

n5.createGroup("a");
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
n5.createGroup("a/a");
assertEquals(++expectedExistCount, n5.getExistCallCount());
expectedAttributeCount+=2;
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
n5.createGroup("a/b");
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
n5.createGroup("a/c");
assertEquals(++expectedExistCount, n5.getExistCallCount());
expectedAttributeCount+=2;
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());

assertArrayEquals(new String[] {"a", "b", "c"}, n5.list("a")); // call list
assertEquals(expectedGroupCount, n5.getGroupCallCount());
Expand Down

0 comments on commit ddffae4

Please sign in to comment.