From ddffae485504218de9a121f303bb8476a2c6d56e Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Tue, 3 Dec 2024 16:26:10 -0500 Subject: [PATCH 1/2] wip/fix: toward avoiding list and exists * cache logic updated * tests updated --- .../n5/zarr/cache/ZarrJsonCache.java | 32 +++++++++++++++++-- .../saalfeldlab/n5/zarr/ZarrCachedFSTest.java | 27 +++++++++++----- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/cache/ZarrJsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/cache/ZarrJsonCache.java index 4102414..bcaaaa1 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/cache/ZarrJsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/cache/ZarrJsonCache.java @@ -17,7 +17,7 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null ){ - addNewCacheInfo(normalPathKey, normalCacheKey, uncachedAttributes ); + addNewCacheInfo(normalPathKey, normalCacheKey, uncachedAttributes); return; } @@ -81,7 +81,7 @@ public boolean isDataset(final String normalPathKey, final String normalCacheKey N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { - addNewCacheInfo(normalPathKey, normalCacheKey, null); + cacheGroupAndDataset(normalPathKey); cacheInfo = getCacheInfo(normalPathKey); } else if (cacheInfo == emptyCacheInfo || cacheInfo.isGroup()) @@ -100,7 +100,7 @@ public boolean isGroup(final String normalPathKey, final String normalCacheKey) N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); if (cacheInfo == null) { - addNewCacheInfo(normalPathKey, normalCacheKey, null); + cacheGroupAndDataset(normalPathKey); cacheInfo = getCacheInfo(normalPathKey); } else if (cacheInfo == emptyCacheInfo || cacheInfo.isDataset()) @@ -114,4 +114,30 @@ else if (!cacheInfo.containsKey(ZarrKeyValueReader.ZGROUP_FILE)) { return cacheInfo.isGroup(); } + public N5CacheInfo cacheGroupAndDataset(final String normalPathKey) { + + final JsonElement zgroup = container.getAttributesFromContainer(normalPathKey, ZarrKeyValueReader.ZGROUP_FILE); + final boolean isGroup = container.isGroupFromAttributes(normalPathKey, zgroup); + + final JsonElement zarray = container.getAttributesFromContainer(normalPathKey, ZarrKeyValueReader.ZARRAY_FILE); + final boolean isDataset = container.isGroupFromAttributes(normalPathKey, zarray); + + if( isGroup || isDataset ) { + + final N5CacheInfo cacheInfo = newCacheInfo(); + + updateCacheIsGroup(cacheInfo, isGroup); + updateCacheAttributes(cacheInfo, ZarrKeyValueReader.ZGROUP_FILE, zgroup); + + updateCacheIsDataset(cacheInfo, isDataset); + updateCacheAttributes(cacheInfo, ZarrKeyValueReader.ZARRAY_FILE, zarray); + + updateCache(normalPathKey, cacheInfo); + return cacheInfo; + } + + updateCache(normalPathKey, emptyCacheInfo); + return emptyCacheInfo; + } + } diff --git a/src/test/java/org/janelia/saalfeldlab/n5/zarr/ZarrCachedFSTest.java b/src/test/java/org/janelia/saalfeldlab/n5/zarr/ZarrCachedFSTest.java index d0743da..bc372d6 100644 --- a/src/test/java/org/janelia/saalfeldlab/n5/zarr/ZarrCachedFSTest.java +++ b/src/test/java/org/janelia/saalfeldlab/n5/zarr/ZarrCachedFSTest.java @@ -168,16 +168,17 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) { int expectedExistCount = 0; final int expectedGroupCount = 0; final int expectedDatasetCount = 0; - int expectedAttributeCount = 0; + int expectedAttributeCount = 0; // isGroup and isDataset are called when creating the reader int expectedListCount = 0; boolean exists = n5.exists(groupA); boolean groupExists = n5.groupExists(groupA); boolean datasetExists = n5.datasetExists(groupA); + expectedAttributeCount+=2; // attributes (zarray and zgroup) are called by groupExists and datasetExists assertFalse(exists); // group does not exist assertFalse(groupExists); // group does not exist assertFalse(datasetExists); // dataset does not exist - assertEquals(++expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); assertEquals(expectedGroupCount, n5.getGroupCallCount()); assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); @@ -189,10 +190,11 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) { exists = n5.exists(groupB); groupExists = n5.groupExists(groupB); datasetExists = n5.datasetExists(groupB); + expectedAttributeCount+=2; // attributes (zarray and zgroup) are called by groupExists and datasetExists assertFalse(exists); // group now exists assertFalse(groupExists); // group now exists assertFalse(datasetExists); // dataset does not exist - assertEquals(++expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); assertEquals(expectedGroupCount, n5.getGroupCallCount()); assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); @@ -212,7 +214,8 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) { // should not check existence when creating a group n5.createGroup(cachedGroup); n5.createGroup(cachedGroup); // be annoying - assertEquals(++expectedExistCount, n5.getExistCallCount()); + expectedAttributeCount+=2; // createGroup calls isGroup and isDataset + assertEquals(expectedExistCount, n5.getExistCallCount()); assertEquals(expectedGroupCount, n5.getGroupCallCount()); assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); @@ -271,7 +274,8 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) { */ final String nonExistentGroup = "doesNotExist"; n5.exists(nonExistentGroup); - assertEquals(++expectedExistCount, n5.getExistCallCount()); + expectedAttributeCount+=2; // exists calls isGroup and isDataset + assertEquals(expectedExistCount, n5.getExistCallCount()); assertEquals(expectedGroupCount, n5.getGroupCallCount()); assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); @@ -310,10 +314,11 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) { final String abc = "a/b/c"; // create "a/b/c" n5.createGroup(abc); + expectedAttributeCount+=2; // createGroup calls isGroup and isDataset assertTrue(n5.exists(abc)); assertTrue(n5.groupExists(abc)); assertFalse(n5.datasetExists(abc)); - assertEquals(++expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedExistCount, n5.getExistCallCount()); assertEquals(expectedGroupCount, n5.getGroupCallCount()); assertEquals(expectedDatasetCount, n5.getDatasetCallCount()); assertEquals(expectedAttributeCount, n5.getAttrCallCount()); @@ -362,12 +367,18 @@ public static void zarrCacheBehaviorHelper(final TrackingStorage n5) { n5.createGroup("a"); assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); n5.createGroup("a/a"); - assertEquals(++expectedExistCount, n5.getExistCallCount()); + expectedAttributeCount+=2; + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); n5.createGroup("a/b"); assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); n5.createGroup("a/c"); - assertEquals(++expectedExistCount, n5.getExistCallCount()); + expectedAttributeCount+=2; + assertEquals(expectedExistCount, n5.getExistCallCount()); + assertEquals(expectedAttributeCount, n5.getAttrCallCount()); assertArrayEquals(new String[] {"a", "b", "c"}, n5.list("a")); // call list assertEquals(expectedGroupCount, n5.getGroupCallCount()); From a6c6d7c64802699fcba69b16adec4d80c14c62f6 Mon Sep 17 00:00:00 2001 From: John Bogovic Date: Fri, 13 Dec 2024 14:13:31 -0500 Subject: [PATCH 2/2] fix: cache methods for avoiding list checks * deprecate the unused method: forceAddNewCacheInfo --- .../n5/zarr/cache/ZarrJsonCache.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/main/java/org/janelia/saalfeldlab/n5/zarr/cache/ZarrJsonCache.java b/src/main/java/org/janelia/saalfeldlab/n5/zarr/cache/ZarrJsonCache.java index bcaaaa1..060262e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/zarr/cache/ZarrJsonCache.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/zarr/cache/ZarrJsonCache.java @@ -52,6 +52,7 @@ public void updateCacheInfo(final String normalPathKey, final String normalCache updateCache(normalPathKey, cacheInfo); } + @Deprecated public N5CacheInfo forceAddNewCacheInfo(final String normalPathKey, final String normalCacheKey, final JsonElement uncachedAttributes, final boolean isGroup, final boolean isDataset) { @@ -76,6 +77,28 @@ public N5CacheInfo forceAddNewCacheInfo(final String normalPathKey, final String return cacheInfo; } + @Override + public JsonElement getAttributes(final String normalPathKey, final String normalCacheKey) { + + N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo == null) { + cacheGroupAndDataset(normalPathKey); + cacheInfo = getCacheInfo(normalPathKey); + } + + if (cacheInfo == emptyCacheInfo || cacheInfo.getCache(normalCacheKey) == emptyJson) { + return null; + } + synchronized (cacheInfo) { + if (!cacheInfo.containsKey(normalCacheKey)) { + updateCacheInfo(normalPathKey, normalCacheKey, null); + } + } + + final JsonElement output = cacheInfo.getCache(normalCacheKey); + return output == null ? null : output.deepCopy(); + } + @Override public boolean isDataset(final String normalPathKey, final String normalCacheKey) { @@ -140,4 +163,16 @@ public N5CacheInfo cacheGroupAndDataset(final String normalPathKey) { return emptyCacheInfo; } + public N5CacheInfo cacheAttributes(final String normalPathKey, final String normalCacheKey) { + + final N5CacheInfo cacheInfo = getCacheInfo(normalPathKey); + if (cacheInfo != null) { + final JsonElement zattrs = container.getAttributesFromContainer(normalPathKey, + normalCacheKey); + updateCacheAttributes(cacheInfo, normalCacheKey, zattrs); + } + + return cacheInfo; + } + }