Skip to content

Commit

Permalink
Merge pull request #57 from saalfeldlab/cache-avoid-ls
Browse files Browse the repository at this point in the history 
Toward avoiding list and exists
  • Loading branch information
@bogovicj
bogovicj authored Dec 13, 2024
2 parents cc42161 + a6c6d7c commit c4885d8
Show file tree
Hide file tree
Showing 2 changed files with 83 additions and 11 deletions.
67 changes: 64 additions & 3 deletions src/main/java/org/janelia/saalfeldlab/n5/zarr/cache/ZarrJsonCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache

N5CacheInfo cacheInfo = getCacheInfo(normalPathKey);
if (cacheInfo == null ){
addNewCacheInfo(normalPathKey, normalCacheKey, uncachedAttributes );
addNewCacheInfo(normalPathKey, normalCacheKey, uncachedAttributes);
return;
}

Expand Down Expand Up @@ -52,6 +52,7 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache
updateCache(normalPathKey, cacheInfo);
}

@Deprecated
public N5CacheInfo forceAddNewCacheInfo(final String normalPathKey, final String normalCacheKey, final JsonElement uncachedAttributes,
final boolean isGroup, final boolean isDataset) {

Expand All @@ -76,12 +77,34 @@ public N5CacheInfo forceAddNewCacheInfo(final String normalPathKey, final String
return cacheInfo;
}

@Override
public JsonElement getAttributes(final String normalPathKey, final String normalCacheKey) {

N5CacheInfo cacheInfo = getCacheInfo(normalPathKey);
if (cacheInfo == null) {
cacheGroupAndDataset(normalPathKey);
cacheInfo = getCacheInfo(normalPathKey);
}

if (cacheInfo == emptyCacheInfo || cacheInfo.getCache(normalCacheKey) == emptyJson) {
return null;
}
synchronized (cacheInfo) {
if (!cacheInfo.containsKey(normalCacheKey)) {
updateCacheInfo(normalPathKey, normalCacheKey, null);
}
}

final JsonElement output = cacheInfo.getCache(normalCacheKey);
return output == null ? null : output.deepCopy();
}

@Override
public boolean isDataset(final String normalPathKey, final String normalCacheKey) {

N5CacheInfo cacheInfo = getCacheInfo(normalPathKey);
if (cacheInfo == null) {
addNewCacheInfo(normalPathKey, normalCacheKey, null);
cacheGroupAndDataset(normalPathKey);
cacheInfo = getCacheInfo(normalPathKey);
}
else if (cacheInfo == emptyCacheInfo || cacheInfo.isGroup())
Expand All @@ -100,7 +123,7 @@ public boolean isGroup(final String normalPathKey, final String normalCacheKey)

N5CacheInfo cacheInfo = getCacheInfo(normalPathKey);
if (cacheInfo == null) {
addNewCacheInfo(normalPathKey, normalCacheKey, null);
cacheGroupAndDataset(normalPathKey);
cacheInfo = getCacheInfo(normalPathKey);
}
else if (cacheInfo == emptyCacheInfo || cacheInfo.isDataset())
Expand All @@ -114,4 +137,42 @@ else if (!cacheInfo.containsKey(ZarrKeyValueReader.ZGROUP_FILE)) {
return cacheInfo.isGroup();
}

public N5CacheInfo cacheGroupAndDataset(final String normalPathKey) {

final JsonElement zgroup = container.getAttributesFromContainer(normalPathKey, ZarrKeyValueReader.ZGROUP_FILE);
final boolean isGroup = container.isGroupFromAttributes(normalPathKey, zgroup);

final JsonElement zarray = container.getAttributesFromContainer(normalPathKey, ZarrKeyValueReader.ZARRAY_FILE);
final boolean isDataset = container.isGroupFromAttributes(normalPathKey, zarray);

if( isGroup || isDataset ) {

final N5CacheInfo cacheInfo = newCacheInfo();

updateCacheIsGroup(cacheInfo, isGroup);
updateCacheAttributes(cacheInfo, ZarrKeyValueReader.ZGROUP_FILE, zgroup);

updateCacheIsDataset(cacheInfo, isDataset);
updateCacheAttributes(cacheInfo, ZarrKeyValueReader.ZARRAY_FILE, zarray);

updateCache(normalPathKey, cacheInfo);
return cacheInfo;
}

updateCache(normalPathKey, emptyCacheInfo);
return emptyCacheInfo;
}

public N5CacheInfo cacheAttributes(final String normalPathKey, final String normalCacheKey) {

final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey);
if (cacheInfo != null) {
final JsonElement zattrs = container.getAttributesFromContainer(normalPathKey,
normalCacheKey);
updateCacheAttributes(cacheInfo, normalCacheKey, zattrs);
}

return cacheInfo;
}

}
27 changes: 19 additions & 8 deletions src/test/java/org/janelia/saalfeldlab/n5/zarr/ZarrCachedFSTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,16 +168,17 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {
int expectedExistCount = 0;
final int expectedGroupCount = 0;
final int expectedDatasetCount = 0;
int expectedAttributeCount = 0;
int expectedAttributeCount = 0; // isGroup and isDataset are called when creating the reader
int expectedListCount = 0;

boolean exists = n5.exists(groupA);
boolean groupExists = n5.groupExists(groupA);
boolean datasetExists = n5.datasetExists(groupA);
expectedAttributeCount+=2; // attributes (zarray and zgroup) are called by groupExists and datasetExists
assertFalse(exists); // group does not exist
assertFalse(groupExists); // group does not exist
assertFalse(datasetExists); // dataset does not exist
assertEquals(++expectedExistCount, n5.getExistCallCount());
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedGroupCount, n5.getGroupCallCount());
assertEquals(expectedDatasetCount, n5.getDatasetCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
Expand All @@ -189,10 +190,11 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {
exists = n5.exists(groupB);
groupExists = n5.groupExists(groupB);
datasetExists = n5.datasetExists(groupB);
expectedAttributeCount+=2; // attributes (zarray and zgroup) are called by groupExists and datasetExists
assertFalse(exists); // group now exists
assertFalse(groupExists); // group now exists
assertFalse(datasetExists); // dataset does not exist
assertEquals(++expectedExistCount, n5.getExistCallCount());
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedGroupCount, n5.getGroupCallCount());
assertEquals(expectedDatasetCount, n5.getDatasetCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
Expand All @@ -212,7 +214,8 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {
// should not check existence when creating a group
n5.createGroup(cachedGroup);
n5.createGroup(cachedGroup); // be annoying
assertEquals(++expectedExistCount, n5.getExistCallCount());
expectedAttributeCount+=2; // createGroup calls isGroup and isDataset
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedGroupCount, n5.getGroupCallCount());
assertEquals(expectedDatasetCount, n5.getDatasetCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
Expand Down Expand Up @@ -271,7 +274,8 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {
*/
final String nonExistentGroup = "doesNotExist";
n5.exists(nonExistentGroup);
assertEquals(++expectedExistCount, n5.getExistCallCount());
expectedAttributeCount+=2; // exists calls isGroup and isDataset
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedGroupCount, n5.getGroupCallCount());
assertEquals(expectedDatasetCount, n5.getDatasetCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
Expand Down Expand Up @@ -310,10 +314,11 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {
final String abc = "a/b/c";
// create "a/b/c"
n5.createGroup(abc);
expectedAttributeCount+=2; // createGroup calls isGroup and isDataset
assertTrue(n5.exists(abc));
assertTrue(n5.groupExists(abc));
assertFalse(n5.datasetExists(abc));
assertEquals(++expectedExistCount, n5.getExistCallCount());
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedGroupCount, n5.getGroupCallCount());
assertEquals(expectedDatasetCount, n5.getDatasetCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
Expand Down Expand Up @@ -362,12 +367,18 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {

n5.createGroup("a");
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
n5.createGroup("a/a");
assertEquals(++expectedExistCount, n5.getExistCallCount());
expectedAttributeCount+=2;
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
n5.createGroup("a/b");
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
n5.createGroup("a/c");
assertEquals(++expectedExistCount, n5.getExistCallCount());
expectedAttributeCount+=2;
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());

assertArrayEquals(new String[] {"a", "b", "c"}, n5.list("a")); // call list
assertEquals(expectedGroupCount, n5.getGroupCallCount());
Expand Down

0 comments on commit c4885d8

Please sign in to comment.