From 689d0f0cf320f40420cf8d875cc758bc4497a46c Mon Sep 17 00:00:00 2001 From: Brian Sam-Bodden Date: Tue, 9 Jul 2024 20:41:02 -0700 Subject: [PATCH] feature: deleteAll - default find/del keys in batches pipelined, optional drop index with dd and rebuild --- .../spring/RedisEnhancedKeyValueAdapter.java | 28 +++++++++++++++++-- .../om/spring/RedisJSONKeyValueAdapter.java | 27 ++++++++++++++++-- .../redis/om/spring/RedisOMProperties.java | 18 ++++++++++++ .../SimpleRedisEnhancedRepository.java | 2 +- 4 files changed, 70 insertions(+), 5 deletions(-) diff --git a/redis-om-spring/src/main/java/com/redis/om/spring/RedisEnhancedKeyValueAdapter.java b/redis-om-spring/src/main/java/com/redis/om/spring/RedisEnhancedKeyValueAdapter.java index 8d4a9e64..31a5b011 100644 --- a/redis-om-spring/src/main/java/com/redis/om/spring/RedisEnhancedKeyValueAdapter.java +++ b/redis-om-spring/src/main/java/com/redis/om/spring/RedisEnhancedKeyValueAdapter.java @@ -11,6 +11,7 @@ import org.springframework.data.convert.CustomConversions; import org.springframework.data.mapping.PersistentPropertyAccessor; import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.connection.RedisKeyCommands; import org.springframework.data.redis.core.*; import org.springframework.data.redis.core.PartialUpdate.PropertyUpdate; import org.springframework.data.redis.core.PartialUpdate.UpdateCommand; @@ -25,6 +26,7 @@ import org.springframework.util.Assert; import org.springframework.util.CollectionUtils; import org.springframework.util.ObjectUtils; +import redis.clients.jedis.commands.KeyCommands; import redis.clients.jedis.search.Query; import redis.clients.jedis.search.SearchResult; @@ -210,8 +212,30 @@ public void deleteAllOf(String keyspace) { Class type = indexer.getEntityClassForKeyspace(keyspace); String searchIndex = indexer.getIndexName(keyspace); SearchOperations searchOps = modulesOperations.opsForSearch(searchIndex); - searchOps.dropIndexAndDocuments(); - indexer.createIndexFor(type); + if (redisOMProperties.getRepository().isDropAndRecreateIndexOnDeleteAll()) { + searchOps.dropIndexAndDocuments(); + indexer.createIndexFor(type); + } else { + boolean moreRecords = true; + while (moreRecords) { + Query query = new Query("*"); + query.limit(0, redisOMProperties.getRepository().getDeleteBatchSize()); + query.setNoContent(); + SearchResult searchResult = searchOps.search(query); + if (searchResult.getTotalResults() > 0) { + List keys = searchResult.getDocuments().stream().map(k -> toBytes(k.getId())).toList(); + redisOperations.executePipelined((RedisCallback) connection -> { + RedisKeyCommands keyCommands = connection.keyCommands(); + for (byte[] key : keys) { + keyCommands.del(key); + } + return null; + }); + } else { + moreRecords = false; + } + } + } } public List getAllIds(String keyspace, Class type) { diff --git a/redis-om-spring/src/main/java/com/redis/om/spring/RedisJSONKeyValueAdapter.java b/redis-om-spring/src/main/java/com/redis/om/spring/RedisJSONKeyValueAdapter.java index 8c497356..325d3db9 100644 --- a/redis-om-spring/src/main/java/com/redis/om/spring/RedisJSONKeyValueAdapter.java +++ b/redis-om-spring/src/main/java/com/redis/om/spring/RedisJSONKeyValueAdapter.java @@ -18,6 +18,7 @@ import org.springframework.dao.OptimisticLockingFailureException; import org.springframework.data.annotation.Reference; import org.springframework.data.annotation.Version; +import org.springframework.data.redis.connection.RedisKeyCommands; import org.springframework.data.redis.core.RedisCallback; import org.springframework.data.redis.core.RedisKeyValueAdapter; import org.springframework.data.redis.core.RedisOperations; @@ -212,8 +213,30 @@ public void deleteAllOf(String keyspace) { Class type = indexer.getEntityClassForKeyspace(keyspace); String searchIndex = indexer.getIndexName(keyspace); SearchOperations searchOps = modulesOperations.opsForSearch(searchIndex); - searchOps.dropIndexAndDocuments(); - indexer.createIndexFor(type); + if (redisOMProperties.getRepository().isDropAndRecreateIndexOnDeleteAll()) { + searchOps.dropIndexAndDocuments(); + indexer.createIndexFor(type); + } else { + boolean moreRecords = true; + while (moreRecords) { + Query query = new Query("*"); + query.limit(0, redisOMProperties.getRepository().getDeleteBatchSize()); + query.setNoContent(); + SearchResult searchResult = searchOps.search(query); + if (searchResult.getTotalResults() > 0) { + List keys = searchResult.getDocuments().stream().map(k -> toBytes(k.getId())).toList(); + redisOperations.executePipelined((RedisCallback) connection -> { + RedisKeyCommands keyCommands = connection.keyCommands(); + for (byte[] key : keys) { + keyCommands.del(key); + } + return null; + }); + } else { + moreRecords = false; + } + } + } } /* diff --git a/redis-om-spring/src/main/java/com/redis/om/spring/RedisOMProperties.java b/redis-om-spring/src/main/java/com/redis/om/spring/RedisOMProperties.java index 242d87f9..9e44d64d 100644 --- a/redis-om-spring/src/main/java/com/redis/om/spring/RedisOMProperties.java +++ b/redis-om-spring/src/main/java/com/redis/om/spring/RedisOMProperties.java @@ -65,11 +65,29 @@ public Ollama getOllama() { public static class Repository { private final Query query = new Query(); + private boolean dropAndRecreateIndexOnDeleteAll = false; + private int deleteBatchSize = 500; public Query getQuery() { return query; } + public boolean isDropAndRecreateIndexOnDeleteAll() { + return dropAndRecreateIndexOnDeleteAll; + } + + public void setDropAndRecreateIndexOnDeleteAll(boolean dropAndRecreateIndexOnDeleteAll) { + this.dropAndRecreateIndexOnDeleteAll = dropAndRecreateIndexOnDeleteAll; + } + + public int getDeleteBatchSize() { + return deleteBatchSize; + } + + public void setDeleteBatchSize(int deleteBatchSize) { + this.deleteBatchSize = deleteBatchSize; + } + public static class Query { private int limit = MAX_SEARCH_RESULTS; private double defaultDistance = DEFAULT_DISTANCE; diff --git a/redis-om-spring/src/main/java/com/redis/om/spring/repository/support/SimpleRedisEnhancedRepository.java b/redis-om-spring/src/main/java/com/redis/om/spring/repository/support/SimpleRedisEnhancedRepository.java index e6812dfa..eaec7348 100644 --- a/redis-om-spring/src/main/java/com/redis/om/spring/repository/support/SimpleRedisEnhancedRepository.java +++ b/redis-om-spring/src/main/java/com/redis/om/spring/repository/support/SimpleRedisEnhancedRepository.java @@ -103,7 +103,7 @@ public Iterable getIds() { Optional maybeIdField = ObjectUtils.getIdFieldForEntityClass(metadata.getJavaType()); String idField = maybeIdField.map(Field::getName).orElse("id"); Query query = new Query("*"); - query.limit(0, MAX_SEARCH_RESULTS); + query.limit(0, properties.getRepository().getQuery().getLimit()); query.returnFields(idField); SearchResult searchResult = searchOps.search(query);