diff --git a/dora/core/common/src/main/java/alluxio/collections/LockPool.java b/dora/core/common/src/main/java/alluxio/collections/LockPool.java index c3dfb5354f61..5c7a75470d2f 100644 --- a/dora/core/common/src/main/java/alluxio/collections/LockPool.java +++ b/dora/core/common/src/main/java/alluxio/collections/LockPool.java @@ -16,6 +16,7 @@ import alluxio.resource.LockResource; import alluxio.resource.RWLockResource; import alluxio.resource.RefCountLockResource; +import alluxio.util.AlluxioFaultInjector; import alluxio.util.ThreadFactoryUtils; import com.google.common.annotations.VisibleForTesting; @@ -165,8 +166,12 @@ private void awaitAndEvict() throws InterruptedException { candidate.mIsAccessed = false; } else { if (candidate.mRefCount.compareAndSet(0, Integer.MIN_VALUE)) { - mIterator.remove(); - numToEvict--; + AlluxioFaultInjector.get().blockUtilAllocatedNewResource(); + Resource tmpResource = mPool.compute(candidateMapEntry.getKey(), + (k, v) -> v == candidate ? null : v); + if (tmpResource == null) { + numToEvict--; + } } } } diff --git a/dora/core/common/src/main/java/alluxio/util/AlluxioFaultInjector.java b/dora/core/common/src/main/java/alluxio/util/AlluxioFaultInjector.java new file mode 100644 index 000000000000..3c98fcfc3a23 --- /dev/null +++ b/dora/core/common/src/main/java/alluxio/util/AlluxioFaultInjector.java @@ -0,0 +1,38 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package alluxio.util; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Used for injecting faults in Alluxio tests. + * Calls into this are a no-op in production code. + */ +@VisibleForTesting +public class AlluxioFaultInjector { + private static AlluxioFaultInjector instance = new AlluxioFaultInjector(); + + public static AlluxioFaultInjector get() { + return instance; + } + + public static void set(AlluxioFaultInjector injector) { + instance = injector; + } + + /** + * Used as a hook to inject intercept when evicting unused resource from pool. + */ + public void blockUtilAllocatedNewResource() { + + } +} diff --git a/dora/core/common/src/test/java/alluxio/collections/LockPoolTest.java b/dora/core/common/src/test/java/alluxio/collections/LockPoolTest.java index 2bf81bd7c617..b9b4de9f58b6 100644 --- a/dora/core/common/src/test/java/alluxio/collections/LockPoolTest.java +++ b/dora/core/common/src/test/java/alluxio/collections/LockPoolTest.java @@ -16,12 +16,16 @@ import alluxio.concurrent.LockMode; import alluxio.resource.LockResource; +import alluxio.util.AlluxioFaultInjector; import alluxio.util.CommonUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.locks.ReentrantReadWriteLock; /** @@ -141,4 +145,53 @@ public void referencedLockTest() { assertTrue(lock1.hasSameLock(mPool.get(50, LockMode.READ))); assertTrue(lock2.hasSameLock(mPool.get(100, LockMode.READ))); } + + @Test(timeout = 10000) + public void evictorTest() throws InterruptedException { + List unClosedLocks = new ArrayList<>(); + CountDownLatch mainThreadDownLatch = new CountDownLatch(1); + CountDownLatch evictorDownLatch = new CountDownLatch(1); + AlluxioFaultInjector.set(new AlluxioFaultInjector() { + public void blockUtilAllocatedNewResource() { + try { + mainThreadDownLatch.countDown(); + evictorDownLatch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }); + + // Acquire 16 locks and just release the lock-3. + for (int i = 1; i <= 16; i++) { + LockResource lock = mPool.get(i, LockMode.READ); + if (i == 3) { + lock.close(); + } else { + unClosedLocks.add(lock); + } + } + assertEquals(16, mPool.size()); + + // Acquire a new Lock to trigger evictor. And the Evictor will release the + // unused Resource-3, but it will be blocked by evictorDownLatch. + LockResource lock17 = mPool.get(17, LockMode.READ); + unClosedLocks.add(lock17); + + // Acquire a resource for key 3 again, currently the Evictor is releasing + // the old Resource-3, but it is blocked right now. + mainThreadDownLatch.await(); + LockResource lock3 = mPool.get(3, LockMode.READ); + unClosedLocks.add(lock3); + + // Allow the evictor to continue. + evictorDownLatch.countDown(); + + // Sleep some millis to wait for the Evictor. + Thread.sleep(100); + + // The Resource 3 should be still in the mPool. + assertEquals(17, mPool.size()); + unClosedLocks.forEach(k -> k.close()); + } }