Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Toward avoiding list and exists #57

Merged
merged 2 commits into from
Dec 13, 2024
Merged

Toward avoiding list and exists #57

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ddffae4
wip/fix: toward avoiding list and exists
bogovicj Dec 3, 2024
a6c6d7c
fix: cache methods for avoiding list checks
bogovicj Dec 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 64 additions & 3 deletions src/main/java/org/janelia/saalfeldlab/n5/zarr/cache/ZarrJsonCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache

N5CacheInfo cacheInfo = getCacheInfo(normalPathKey);
if (cacheInfo == null ){
addNewCacheInfo(normalPathKey, normalCacheKey, uncachedAttributes );
addNewCacheInfo(normalPathKey, normalCacheKey, uncachedAttributes);
return;
}

Expand Down Expand Up @@ -52,6 +52,7 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache
updateCache(normalPathKey, cacheInfo);
}

@Deprecated
public N5CacheInfo forceAddNewCacheInfo(final String normalPathKey, final String normalCacheKey, final JsonElement uncachedAttributes,
final boolean isGroup, final boolean isDataset) {

Expand All @@ -76,12 +77,34 @@ public N5CacheInfo forceAddNewCacheInfo(final String normalPathKey, final String
return cacheInfo;
}

@Override
public JsonElement getAttributes(final String normalPathKey, final String normalCacheKey) {

N5CacheInfo cacheInfo = getCacheInfo(normalPathKey);
if (cacheInfo == null) {
cacheGroupAndDataset(normalPathKey);
cacheInfo = getCacheInfo(normalPathKey);
}

if (cacheInfo == emptyCacheInfo || cacheInfo.getCache(normalCacheKey) == emptyJson) {
return null;
}
synchronized (cacheInfo) {
if (!cacheInfo.containsKey(normalCacheKey)) {
updateCacheInfo(normalPathKey, normalCacheKey, null);
}
}

final JsonElement output = cacheInfo.getCache(normalCacheKey);
return output == null ? null : output.deepCopy();
}

@Override
public boolean isDataset(final String normalPathKey, final String normalCacheKey) {

N5CacheInfo cacheInfo = getCacheInfo(normalPathKey);
if (cacheInfo == null) {
addNewCacheInfo(normalPathKey, normalCacheKey, null);
cacheGroupAndDataset(normalPathKey);
cacheInfo = getCacheInfo(normalPathKey);
}
else if (cacheInfo == emptyCacheInfo || cacheInfo.isGroup())
Expand All @@ -100,7 +123,7 @@ public boolean isGroup(final String normalPathKey, final String normalCacheKey)

N5CacheInfo cacheInfo = getCacheInfo(normalPathKey);
if (cacheInfo == null) {
addNewCacheInfo(normalPathKey, normalCacheKey, null);
cacheGroupAndDataset(normalPathKey);
cacheInfo = getCacheInfo(normalPathKey);
}
else if (cacheInfo == emptyCacheInfo || cacheInfo.isDataset())
Expand All @@ -114,4 +137,42 @@ else if (!cacheInfo.containsKey(ZarrKeyValueReader.ZGROUP_FILE)) {
return cacheInfo.isGroup();
}

public N5CacheInfo cacheGroupAndDataset(final String normalPathKey) {

final JsonElement zgroup = container.getAttributesFromContainer(normalPathKey, ZarrKeyValueReader.ZGROUP_FILE);
final boolean isGroup = container.isGroupFromAttributes(normalPathKey, zgroup);

final JsonElement zarray = container.getAttributesFromContainer(normalPathKey, ZarrKeyValueReader.ZARRAY_FILE);
final boolean isDataset = container.isGroupFromAttributes(normalPathKey, zarray);

if( isGroup || isDataset ) {

final N5CacheInfo cacheInfo = newCacheInfo();

updateCacheIsGroup(cacheInfo, isGroup);
updateCacheAttributes(cacheInfo, ZarrKeyValueReader.ZGROUP_FILE, zgroup);

updateCacheIsDataset(cacheInfo, isDataset);
updateCacheAttributes(cacheInfo, ZarrKeyValueReader.ZARRAY_FILE, zarray);

updateCache(normalPathKey, cacheInfo);
return cacheInfo;
}

updateCache(normalPathKey, emptyCacheInfo);
return emptyCacheInfo;
}

public N5CacheInfo cacheAttributes(final String normalPathKey, final String normalCacheKey) {

final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey);
if (cacheInfo != null) {
final JsonElement zattrs = container.getAttributesFromContainer(normalPathKey,
normalCacheKey);
updateCacheAttributes(cacheInfo, normalCacheKey, zattrs);
}

return cacheInfo;
}

}
27 changes: 19 additions & 8 deletions src/test/java/org/janelia/saalfeldlab/n5/zarr/ZarrCachedFSTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,16 +168,17 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {
int expectedExistCount = 0;
final int expectedGroupCount = 0;
final int expectedDatasetCount = 0;
int expectedAttributeCount = 0;
int expectedAttributeCount = 0; // isGroup and isDataset are called when creating the reader
int expectedListCount = 0;

boolean exists = n5.exists(groupA);
boolean groupExists = n5.groupExists(groupA);
boolean datasetExists = n5.datasetExists(groupA);
expectedAttributeCount+=2; // attributes (zarray and zgroup) are called by groupExists and datasetExists
assertFalse(exists); // group does not exist
assertFalse(groupExists); // group does not exist
assertFalse(datasetExists); // dataset does not exist
assertEquals(++expectedExistCount, n5.getExistCallCount());
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedGroupCount, n5.getGroupCallCount());
assertEquals(expectedDatasetCount, n5.getDatasetCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
Expand All @@ -189,10 +190,11 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {
exists = n5.exists(groupB);
groupExists = n5.groupExists(groupB);
datasetExists = n5.datasetExists(groupB);
expectedAttributeCount+=2; // attributes (zarray and zgroup) are called by groupExists and datasetExists
assertFalse(exists); // group now exists
assertFalse(groupExists); // group now exists
assertFalse(datasetExists); // dataset does not exist
assertEquals(++expectedExistCount, n5.getExistCallCount());
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedGroupCount, n5.getGroupCallCount());
assertEquals(expectedDatasetCount, n5.getDatasetCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
Expand All @@ -212,7 +214,8 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {
// should not check existence when creating a group
n5.createGroup(cachedGroup);
n5.createGroup(cachedGroup); // be annoying
assertEquals(++expectedExistCount, n5.getExistCallCount());
expectedAttributeCount+=2; // createGroup calls isGroup and isDataset
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedGroupCount, n5.getGroupCallCount());
assertEquals(expectedDatasetCount, n5.getDatasetCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
Expand Down Expand Up @@ -271,7 +274,8 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {
*/
final String nonExistentGroup = "doesNotExist";
n5.exists(nonExistentGroup);
assertEquals(++expectedExistCount, n5.getExistCallCount());
expectedAttributeCount+=2; // exists calls isGroup and isDataset
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedGroupCount, n5.getGroupCallCount());
assertEquals(expectedDatasetCount, n5.getDatasetCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
Expand Down Expand Up @@ -310,10 +314,11 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {
final String abc = "a/b/c";
// create "a/b/c"
n5.createGroup(abc);
expectedAttributeCount+=2; // createGroup calls isGroup and isDataset
assertTrue(n5.exists(abc));
assertTrue(n5.groupExists(abc));
assertFalse(n5.datasetExists(abc));
assertEquals(++expectedExistCount, n5.getExistCallCount());
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedGroupCount, n5.getGroupCallCount());
assertEquals(expectedDatasetCount, n5.getDatasetCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
Expand Down Expand Up @@ -362,12 +367,18 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) {

n5.createGroup("a");
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
n5.createGroup("a/a");
assertEquals(++expectedExistCount, n5.getExistCallCount());
expectedAttributeCount+=2;
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
n5.createGroup("a/b");
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());
n5.createGroup("a/c");
assertEquals(++expectedExistCount, n5.getExistCallCount());
expectedAttributeCount+=2;
assertEquals(expectedExistCount, n5.getExistCallCount());
assertEquals(expectedAttributeCount, n5.getAttrCallCount());

assertArrayEquals(new String[] {"a", "b", "c"}, n5.list("a")); // call list
assertEquals(expectedGroupCount, n5.getGroupCallCount());
Expand Down
Loading