diff --git a/src/storage/knn_index/knn_ivf/ivf_index_data_in_mem.cpp b/src/storage/knn_index/knn_ivf/ivf_index_data_in_mem.cpp
index 40c8353af4..9d7f54eb19 100644
--- a/src/storage/knn_index/knn_ivf/ivf_index_data_in_mem.cpp
+++ b/src/storage/knn_index/knn_ivf/ivf_index_data_in_mem.cpp
@@ -110,7 +110,11 @@ class IVFIndexInMemT final : public IVFIndexInMem {
         }
     }
 
-    ~IVFIndexInMemT() { BaseMemIndex::DecreaseMemoryUsage(MemoryUsed()); }
+    ~IVFIndexInMemT() {
+        if (own_ivf_index_storage_) {
+            BaseMemIndex::DecreaseMemoryUsage(MemoryUsed());
+        }
+    }
 
     MemIndexTracerInfo GetInfo() const override {
         auto *table_index_entry = segment_index_entry_->table_index_entry();