diff --git a/dump.txt b/dump.txt deleted file mode 100644 index 49380ece6278d..0000000000000 --- a/dump.txt +++ /dev/null @@ -1,690 +0,0 @@ -2023-08-31 13:32:30 -Full thread dump OpenJDK 64-Bit Server VM (20.0.1+9-29 mixed mode, sharing): - -Threads class SMR info: -_java_thread_list=0x00007fa110000b60, length=40, elements={ -0x00007fa2880340f0, 0x00007fa28830fb90, 0x00007fa2883110c0, 0x00007fa288312a80, -0x00007fa288313f10, 0x00007fa288315390, 0x00007fa288316e50, 0x00007fa288318430, -0x00007fa288366990, 0x00007fa288368cf0, 0x00007fa288d38f50, 0x00007fa288d581d0, -0x00007fa288d58f80, 0x00007fa288f7e850, 0x00007fa1d46dc970, 0x00007fa194754b00, -0x00007fa195842360, 0x00007fa1958ae1a0, 0x00007fa1958f2de0, 0x00007fa1880db260, -0x00007fa1880dbb20, 0x00007fa18c12ca90, 0x00007fa18c1b23c0, 0x00007fa18815fcd0, -0x00007fa18c19a640, 0x00007fa18c1a0870, 0x00007fa18c1a19f0, 0x00007fa18c1a2b30, -0x00007fa18c08c510, 0x00007fa18c08d570, 0x00007fa18c090980, 0x00007fa18c0919e0, -0x00007fa1958fcc90, 0x00007fa18c0f4870, 0x00007fa23c000e70, 0x00007fa1880777c0, -0x00007fa1c8070050, 0x00007fa2202ef430, 0x00007fa220001080, 0x00007fa2181ddaa0 -} - -"Test worker" #1 [437080] prio=5 os_prio=0 cpu=384.56ms elapsed=1208.66s tid=0x00007fa2880340f0 nid=437080 in Object.wait() [0x00007fa28d5fe000] - java.lang.Thread.State: WAITING (on object monitor) - at java.lang.Object.wait0(java.base@20.0.1/Native Method) - - waiting on <0x00000000eca00000> (a com.carrotsearch.randomizedtesting.RandomizedRunner$2) - at java.lang.Object.wait(java.base@20.0.1/Object.java:366) - at java.lang.Thread.join(java.base@20.0.1/Thread.java:2115) - - locked <0x00000000eca00000> (a com.carrotsearch.randomizedtesting.RandomizedRunner$2) - at java.lang.Thread.join(java.base@20.0.1/Thread.java:2191) - at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:647) - at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:504) - at org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.runTestClass(JUnitTestClassExecutor.java:108) - at org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:58) - at org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:40) - at org.gradle.api.internal.tasks.testing.junit.AbstractJUnitTestClassProcessor.processTestClass(AbstractJUnitTestClassProcessor.java:60) - at org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:52) - at java.lang.invoke.LambdaForm$DMH/0x0000000801003400.invokeInterface(java.base@20.0.1/LambdaForm$DMH) - at java.lang.invoke.LambdaForm$MH/0x0000000801016c00.invoke(java.base@20.0.1/LambdaForm$MH) - at java.lang.invoke.Invokers$Holder.invokeExact_MT(java.base@20.0.1/Invokers$Holder) - at jdk.internal.reflect.DirectMethodHandleAccessor.invokeImpl(java.base@20.0.1/DirectMethodHandleAccessor.java:155) - at jdk.internal.reflect.DirectMethodHandleAccessor.invoke(java.base@20.0.1/DirectMethodHandleAccessor.java:104) - at java.lang.reflect.Method.invoke(java.base@20.0.1/Method.java:578) - at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36) - at org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) - at org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33) - at org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94) - at jdk.proxy1.$Proxy2.processTestClass(jdk.proxy1/Unknown Source) - at org.gradle.api.internal.tasks.testing.worker.TestWorker$2.run(TestWorker.java:176) - at org.gradle.api.internal.tasks.testing.worker.TestWorker.executeAndMaintainThreadName(TestWorker.java:129) - at org.gradle.api.internal.tasks.testing.worker.TestWorker.execute(TestWorker.java:100) - at org.gradle.api.internal.tasks.testing.worker.TestWorker.execute(TestWorker.java:60) - at org.gradle.process.internal.worker.child.ActionExecutionWorker.execute(ActionExecutionWorker.java:56) - at org.gradle.process.internal.worker.child.SystemApplicationClassLoaderWorker.call(SystemApplicationClassLoaderWorker.java:113) - at org.gradle.process.internal.worker.child.SystemApplicationClassLoaderWorker.call(SystemApplicationClassLoaderWorker.java:65) - at worker.org.gradle.process.internal.worker.GradleWorkerMain.run(GradleWorkerMain.java:69) - at worker.org.gradle.process.internal.worker.GradleWorkerMain.main(GradleWorkerMain.java:74) - -"Reference Handler" #8 [437087] daemon prio=10 os_prio=0 cpu=4.73ms elapsed=1208.64s tid=0x00007fa28830fb90 nid=437087 waiting on condition [0x00007fa25d213000] - java.lang.Thread.State: RUNNABLE - at java.lang.ref.Reference.waitForReferencePendingList(java.base@20.0.1/Native Method) - at java.lang.ref.Reference.processPendingReferences(java.base@20.0.1/Reference.java:246) - at java.lang.ref.Reference$ReferenceHandler.run(java.base@20.0.1/Reference.java:208) - -"Finalizer" #9 [437088] daemon prio=8 os_prio=0 cpu=0.61ms elapsed=1208.64s tid=0x00007fa2883110c0 nid=437088 in Object.wait() [0x00007fa25d112000] - java.lang.Thread.State: WAITING (on object monitor) - at java.lang.Object.wait0(java.base@20.0.1/Native Method) - - waiting on <0x00000000e0b9ce80> (a java.lang.ref.NativeReferenceQueue$Lock) - at java.lang.Object.wait(java.base@20.0.1/Object.java:366) - at java.lang.Object.wait(java.base@20.0.1/Object.java:339) - at java.lang.ref.NativeReferenceQueue.await(java.base@20.0.1/NativeReferenceQueue.java:48) - at java.lang.ref.ReferenceQueue.remove0(java.base@20.0.1/ReferenceQueue.java:158) - at java.lang.ref.NativeReferenceQueue.remove(java.base@20.0.1/NativeReferenceQueue.java:89) - - locked <0x00000000e0b9ce80> (a java.lang.ref.NativeReferenceQueue$Lock) - at java.lang.ref.Finalizer$FinalizerThread.run(java.base@20.0.1/Finalizer.java:173) - -"Signal Dispatcher" #10 [437089] daemon prio=9 os_prio=0 cpu=0.37ms elapsed=1208.64s tid=0x00007fa288312a80 nid=437089 waiting on condition [0x0000000000000000] - java.lang.Thread.State: RUNNABLE - -"Service Thread" #11 [437090] daemon prio=9 os_prio=0 cpu=10.86ms elapsed=1208.64s tid=0x00007fa288313f10 nid=437090 runnable [0x0000000000000000] - java.lang.Thread.State: RUNNABLE - -"Monitor Deflation Thread" #12 [437091] daemon prio=9 os_prio=0 cpu=17.44ms elapsed=1208.64s tid=0x00007fa288315390 nid=437091 runnable [0x0000000000000000] - java.lang.Thread.State: RUNNABLE - -"C2 CompilerThread0" #13 [437092] daemon prio=9 os_prio=0 cpu=4310.74ms elapsed=1208.64s tid=0x00007fa288316e50 nid=437092 waiting on condition [0x0000000000000000] - java.lang.Thread.State: RUNNABLE - No compile task - -"C1 CompilerThread0" #25 [437093] daemon prio=9 os_prio=0 cpu=863.79ms elapsed=1208.64s tid=0x00007fa288318430 nid=437093 waiting on condition [0x0000000000000000] - java.lang.Thread.State: RUNNABLE - No compile task - -"Common-Cleaner" #31 [437095] daemon prio=8 os_prio=0 cpu=7.24ms elapsed=1208.62s tid=0x00007fa288366990 nid=437095 waiting on condition [0x00007fa25ca0b000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000e0b9e6b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@20.0.1/AbstractQueuedSynchronizer.java:1847) - at java.lang.ref.ReferenceQueue.await(java.base@20.0.1/ReferenceQueue.java:71) - at java.lang.ref.ReferenceQueue.remove0(java.base@20.0.1/ReferenceQueue.java:143) - at java.lang.ref.ReferenceQueue.remove(java.base@20.0.1/ReferenceQueue.java:218) - at jdk.internal.ref.CleanerImpl.run(java.base@20.0.1/CleanerImpl.java:140) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - at jdk.internal.misc.InnocuousThread.run(java.base@20.0.1/InnocuousThread.java:186) - -"Notification Thread" #32 [437096] daemon prio=9 os_prio=0 cpu=0.08ms elapsed=1208.62s tid=0x00007fa288368cf0 nid=437096 runnable [0x0000000000000000] - java.lang.Thread.State: RUNNABLE - -"/127.0.0.1:56514 to /127.0.0.1:44791 workers" #35 [437106] prio=5 os_prio=0 cpu=3.41ms elapsed=1208.37s tid=0x00007fa288d38f50 nid=437106 waiting on condition [0x00007fa25c2f3000] - java.lang.Thread.State: WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000e0b9ff00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) - at java.util.concurrent.locks.LockSupport.park(java.base@20.0.1/LockSupport.java:371) - at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(java.base@20.0.1/AbstractQueuedSynchronizer.java:519) - at java.util.concurrent.ForkJoinPool.unmanagedBlock(java.base@20.0.1/ForkJoinPool.java:3780) - at java.util.concurrent.ForkJoinPool.managedBlock(java.base@20.0.1/ForkJoinPool.java:3725) - at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@20.0.1/AbstractQueuedSynchronizer.java:1707) - at org.gradle.internal.remote.internal.hub.queue.EndPointQueue.take(EndPointQueue.java:49) - at org.gradle.internal.remote.internal.hub.MessageHub$Handler.run(MessageHub.java:403) - at org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64) - at org.gradle.internal.concurrent.AbstractManagedExecutor$1.run(AbstractManagedExecutor.java:47) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1144) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"/127.0.0.1:56514 to /127.0.0.1:44791 workers Thread 2" #36 [437107] prio=5 os_prio=0 cpu=28.88ms elapsed=1208.36s tid=0x00007fa288d581d0 nid=437107 waiting on condition [0x00007fa25c1f2000] - java.lang.Thread.State: WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000e0ba06f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) - at java.util.concurrent.locks.LockSupport.park(java.base@20.0.1/LockSupport.java:371) - at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(java.base@20.0.1/AbstractQueuedSynchronizer.java:519) - at java.util.concurrent.ForkJoinPool.unmanagedBlock(java.base@20.0.1/ForkJoinPool.java:3780) - at java.util.concurrent.ForkJoinPool.managedBlock(java.base@20.0.1/ForkJoinPool.java:3725) - at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@20.0.1/AbstractQueuedSynchronizer.java:1707) - at org.gradle.internal.remote.internal.hub.queue.EndPointQueue.take(EndPointQueue.java:49) - at org.gradle.internal.remote.internal.hub.MessageHub$ConnectionDispatch.run(MessageHub.java:322) - at org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64) - at org.gradle.internal.concurrent.AbstractManagedExecutor$1.run(AbstractManagedExecutor.java:47) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1144) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"/127.0.0.1:56514 to /127.0.0.1:44791 workers Thread 3" #37 [437108] prio=5 os_prio=0 cpu=6.14ms elapsed=1208.36s tid=0x00007fa288d58f80 nid=437108 runnable [0x00007fa20bffe000] - java.lang.Thread.State: RUNNABLE - at sun.nio.ch.EPoll.wait(java.base@20.0.1/Native Method) - at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@20.0.1/EPollSelectorImpl.java:121) - at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@20.0.1/SelectorImpl.java:130) - - locked <0x00000000e0b9e000> (a sun.nio.ch.Util$2) - - locked <0x00000000e0b9dfb0> (a sun.nio.ch.EPollSelectorImpl) - at sun.nio.ch.SelectorImpl.select(java.base@20.0.1/SelectorImpl.java:147) - at org.gradle.internal.remote.internal.inet.SocketConnection$SocketInputStream.read(SocketConnection.java:185) - at com.esotericsoftware.kryo.io.Input.fill(Input.java:146) - at com.esotericsoftware.kryo.io.Input.require(Input.java:178) - at com.esotericsoftware.kryo.io.Input.readByte(Input.java:295) - at org.gradle.internal.serialize.kryo.KryoBackedDecoder.readByte(KryoBackedDecoder.java:88) - at org.gradle.internal.remote.internal.hub.InterHubMessageSerializer$MessageReader.read(InterHubMessageSerializer.java:64) - at org.gradle.internal.remote.internal.hub.InterHubMessageSerializer$MessageReader.read(InterHubMessageSerializer.java:52) - at org.gradle.internal.remote.internal.inet.SocketConnection.receive(SocketConnection.java:81) - at org.gradle.internal.remote.internal.hub.MessageHub$ConnectionReceive.run(MessageHub.java:270) - at org.gradle.internal.concurrent.ExecutorPolicy$CatchAndRecordFailures.onExecute(ExecutorPolicy.java:64) - at org.gradle.internal.concurrent.AbstractManagedExecutor$1.run(AbstractManagedExecutor.java:47) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1144) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"SUITE-JwtRealmSingleNodeTests-seed#[76C5752C0DE4858C]" #49 [437120] prio=10 os_prio=0 cpu=977.86ms elapsed=1208.20s tid=0x00007fa288f7e850 nid=437120 waiting on condition [0x00007fa20b3e7000] - java.lang.Thread.State: TIMED_WAITING (sleeping) - at java.lang.Thread.sleep0(java.base@20.0.1/Native Method) - at java.lang.Thread.sleep(java.base@20.0.1/Thread.java:484) - at com.carrotsearch.randomizedtesting.ThreadLeakControl$$Lambda$5064/0x0000000801bafce8.await(Unknown Source) - at com.carrotsearch.randomizedtesting.ThreadLeakControl.join(ThreadLeakControl.java:888) - at com.carrotsearch.randomizedtesting.ThreadLeakControl.tryToInterruptAll(ThreadLeakControl.java:803) - at com.carrotsearch.randomizedtesting.ThreadLeakControl.checkThreadLeaks(ThreadLeakControl.java:649) - at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:461) - at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:716) - at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:138) - at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:637) - -"TEST-JwtRealmSingleNodeTests.testClientSecretRotation-seed#[76C5752C0DE4858C]" #63 [437138] prio=5 os_prio=0 cpu=3256.67ms elapsed=1207.36s tid=0x00007fa1d46dc970 nid=437138 waiting on condition [0x00007fa20a2ae000] - java.lang.Thread.State: WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000eca86a60> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - at java.util.concurrent.locks.LockSupport.park(java.base@20.0.1/LockSupport.java:221) - at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@20.0.1/AbstractQueuedSynchronizer.java:754) - at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(java.base@20.0.1/AbstractQueuedSynchronizer.java:990) - at java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock(java.base@20.0.1/ReentrantReadWriteLock.java:959) - at org.elasticsearch.xpack.security.support.RotatableSecret.checkExpired(RotatableSecret.java:99) - at org.elasticsearch.xpack.security.support.RotatableSecret.innerIsSet(RotatableSecret.java:93) - at org.elasticsearch.xpack.security.support.RotatableSecret.matches(RotatableSecret.java:77) - at org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests.testClientSecretRotation(JwtRealmSingleNodeTests.java:233) - at java.lang.invoke.LambdaForm$DMH/0x0000000801284000.invokeVirtual(java.base@20.0.1/LambdaForm$DMH) - at java.lang.invoke.LambdaForm$MH/0x00000008010a0800.invoke(java.base@20.0.1/LambdaForm$MH) - at java.lang.invoke.Invokers$Holder.invokeExact_MT(java.base@20.0.1/Invokers$Holder) - at jdk.internal.reflect.DirectMethodHandleAccessor.invokeImpl(java.base@20.0.1/DirectMethodHandleAccessor.java:154) - at jdk.internal.reflect.DirectMethodHandleAccessor.invoke(java.base@20.0.1/DirectMethodHandleAccessor.java:104) - at java.lang.reflect.Method.invoke(java.base@20.0.1/Method.java:578) - at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758) - at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946) - at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982) - at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996) - at org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:48) - at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) - at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48) - at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) - at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45) - at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) - at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) - at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) - at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390) - at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843) - at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490) - at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955) - at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840) - at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891) - at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902) - at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) - at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) - at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38) - at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) - at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40) - at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) - at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) - at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53) - at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43) - at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44) - at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60) - at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47) - at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36) - at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390) - at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850) - at com.carrotsearch.randomizedtesting.ThreadLeakControl$$Lambda$517/0x0000000801305ae0.run(Unknown Source) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][scheduler][T#1]" #65 [437145] daemon prio=5 os_prio=0 cpu=453.20ms elapsed=1206.80s tid=0x00007fa194754b00 nid=437145 waiting on condition [0x00007fa209daa000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000ec900070> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@20.0.1/AbstractQueuedSynchronizer.java:1758) - at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@20.0.1/ScheduledThreadPoolExecutor.java:1182) - at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@20.0.1/ScheduledThreadPoolExecutor.java:899) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1070) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][transport_worker][T#1]" #86 [437171] daemon prio=5 os_prio=0 cpu=5.86ms elapsed=1204.08s tid=0x00007fa195842360 nid=437171 runnable [0x00007fa209fac000] - java.lang.Thread.State: RUNNABLE - at sun.nio.ch.EPoll.wait(java.base@20.0.1/Native Method) - at sun.nio.ch.EPollSelectorImpl.doSelect(java.base@20.0.1/EPollSelectorImpl.java:121) - at sun.nio.ch.SelectorImpl.lockAndDoSelect(java.base@20.0.1/SelectorImpl.java:130) - - locked <0x00000000ec986ab0> (a sun.nio.ch.Util$2) - - locked <0x00000000ec986a60> (a sun.nio.ch.EPollSelectorImpl) - at sun.nio.ch.SelectorImpl.select(java.base@20.0.1/SelectorImpl.java:147) - at io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) - at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) - at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) - at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][cluster_coordination][T#1]" #87 [437172] daemon prio=5 os_prio=0 cpu=138.05ms elapsed=1203.98s tid=0x00007fa1958ae1a0 nid=437172 waiting on condition [0x00007fa209397000] - java.lang.Thread.State: WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000ec9c9ff0> (a java.util.concurrent.LinkedTransferQueue) - at java.util.concurrent.locks.LockSupport.park(java.base@20.0.1/LockSupport.java:371) - at java.util.concurrent.LinkedTransferQueue$Node.block(java.base@20.0.1/LinkedTransferQueue.java:470) - at java.util.concurrent.ForkJoinPool.unmanagedBlock(java.base@20.0.1/ForkJoinPool.java:3780) - at java.util.concurrent.ForkJoinPool.managedBlock(java.base@20.0.1/ForkJoinPool.java:3725) - at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@20.0.1/LinkedTransferQueue.java:669) - at java.util.concurrent.LinkedTransferQueue.xfer(java.base@20.0.1/LinkedTransferQueue.java:616) - at java.util.concurrent.LinkedTransferQueue.take(java.base@20.0.1/LinkedTransferQueue.java:1286) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1070) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][clusterApplierService#updateTask][T#1]" #88 [437173] daemon prio=5 os_prio=0 cpu=121.06ms elapsed=1203.93s tid=0x00007fa1958f2de0 nid=437173 waiting on condition [0x00007fa20a0ad000] - java.lang.Thread.State: WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000ec843588> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) - at java.util.concurrent.locks.LockSupport.park(java.base@20.0.1/LockSupport.java:371) - at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(java.base@20.0.1/AbstractQueuedSynchronizer.java:519) - at java.util.concurrent.ForkJoinPool.unmanagedBlock(java.base@20.0.1/ForkJoinPool.java:3780) - at java.util.concurrent.ForkJoinPool.managedBlock(java.base@20.0.1/ForkJoinPool.java:3725) - at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(java.base@20.0.1/AbstractQueuedSynchronizer.java:1707) - at java.util.concurrent.PriorityBlockingQueue.take(java.base@20.0.1/PriorityBlockingQueue.java:535) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1070) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][generic][T#1]" #90 [437175] daemon prio=5 os_prio=0 cpu=51.57ms elapsed=1203.81s tid=0x00007fa1880db260 nid=437175 waiting on condition [0x00007fa209296000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000eca436c0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@20.0.1/LinkedTransferQueue.java:676) - at java.util.concurrent.LinkedTransferQueue.xfer(java.base@20.0.1/LinkedTransferQueue.java:616) - at java.util.concurrent.LinkedTransferQueue.poll(java.base@20.0.1/LinkedTransferQueue.java:1294) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1069) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][generic][T#2]" #91 [437176] daemon prio=5 os_prio=0 cpu=43.18ms elapsed=1203.81s tid=0x00007fa1880dbb20 nid=437176 waiting on condition [0x00007fa20867f000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000eca436c0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@20.0.1/LinkedTransferQueue.java:676) - at java.util.concurrent.LinkedTransferQueue.xfer(java.base@20.0.1/LinkedTransferQueue.java:616) - at java.util.concurrent.LinkedTransferQueue.poll(java.base@20.0.1/LinkedTransferQueue.java:1294) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1069) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][management][T#1]" #93 [437178] daemon prio=5 os_prio=0 cpu=203.27ms elapsed=1203.75s tid=0x00007fa18c12ca90 nid=437178 waiting on condition [0x00007fa20847d000] - java.lang.Thread.State: WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000ecaca0a0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) - at java.util.concurrent.locks.LockSupport.park(java.base@20.0.1/LockSupport.java:371) - at java.util.concurrent.LinkedTransferQueue$Node.block(java.base@20.0.1/LinkedTransferQueue.java:470) - at java.util.concurrent.ForkJoinPool.unmanagedBlock(java.base@20.0.1/ForkJoinPool.java:3780) - at java.util.concurrent.ForkJoinPool.managedBlock(java.base@20.0.1/ForkJoinPool.java:3725) - at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@20.0.1/LinkedTransferQueue.java:669) - at java.util.concurrent.LinkedTransferQueue.xfer(java.base@20.0.1/LinkedTransferQueue.java:616) - at java.util.concurrent.LinkedTransferQueue.take(java.base@20.0.1/LinkedTransferQueue.java:1286) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1070) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"FileSystemWatchService" #94 [437179] daemon prio=5 os_prio=0 cpu=0.62ms elapsed=1203.73s tid=0x00007fa18c1b23c0 nid=437179 runnable [0x00007fa20837c000] - java.lang.Thread.State: RUNNABLE - at sun.nio.fs.LinuxWatchService.poll(java.base@20.0.1/Native Method) - at sun.nio.fs.LinuxWatchService$Poller.run(java.base@20.0.1/LinuxWatchService.java:309) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][generic][T#4]" #96 [437181] daemon prio=5 os_prio=0 cpu=49.51ms elapsed=1203.72s tid=0x00007fa18815fcd0 nid=437181 waiting on condition [0x00007fa20817a000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000eca436c0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@20.0.1/LinkedTransferQueue.java:676) - at java.util.concurrent.LinkedTransferQueue.xfer(java.base@20.0.1/LinkedTransferQueue.java:616) - at java.util.concurrent.LinkedTransferQueue.poll(java.base@20.0.1/LinkedTransferQueue.java:1294) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1069) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][trigger_engine_scheduler][T#1]" #97 [437182] daemon prio=5 os_prio=0 cpu=2.69ms elapsed=1203.71s tid=0x00007fa18c19a640 nid=437182 waiting on condition [0x00007fa10bffe000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000ec9001c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@20.0.1/AbstractQueuedSynchronizer.java:1758) - at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@20.0.1/ScheduledThreadPoolExecutor.java:1182) - at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@20.0.1/ScheduledThreadPoolExecutor.java:899) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1070) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][generic][T#5]" #98 [437183] daemon prio=5 os_prio=0 cpu=82.54ms elapsed=1203.71s tid=0x00007fa18c1a0870 nid=437183 waiting on condition [0x00007fa10befd000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000eca436c0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@20.0.1/LinkedTransferQueue.java:676) - at java.util.concurrent.LinkedTransferQueue.xfer(java.base@20.0.1/LinkedTransferQueue.java:616) - at java.util.concurrent.LinkedTransferQueue.poll(java.base@20.0.1/LinkedTransferQueue.java:1294) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1069) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][generic][T#6]" #99 [437184] daemon prio=5 os_prio=0 cpu=71.77ms elapsed=1203.71s tid=0x00007fa18c1a19f0 nid=437184 waiting on condition [0x00007fa10bdfc000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000eca436c0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@20.0.1/LinkedTransferQueue.java:676) - at java.util.concurrent.LinkedTransferQueue.xfer(java.base@20.0.1/LinkedTransferQueue.java:616) - at java.util.concurrent.LinkedTransferQueue.poll(java.base@20.0.1/LinkedTransferQueue.java:1294) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1069) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][generic][T#7]" #100 [437185] daemon prio=5 os_prio=0 cpu=61.10ms elapsed=1203.71s tid=0x00007fa18c1a2b30 nid=437185 waiting on condition [0x00007fa10bcfb000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000eca436c0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@20.0.1/LinkedTransferQueue.java:676) - at java.util.concurrent.LinkedTransferQueue.xfer(java.base@20.0.1/LinkedTransferQueue.java:616) - at java.util.concurrent.LinkedTransferQueue.poll(java.base@20.0.1/LinkedTransferQueue.java:1294) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1069) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][generic][T#8]" #101 [437186] daemon prio=5 os_prio=0 cpu=52.38ms elapsed=1203.71s tid=0x00007fa18c08c510 nid=437186 waiting on condition [0x00007fa10bbfa000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000eca436c0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@20.0.1/LinkedTransferQueue.java:676) - at java.util.concurrent.LinkedTransferQueue.xfer(java.base@20.0.1/LinkedTransferQueue.java:616) - at java.util.concurrent.LinkedTransferQueue.poll(java.base@20.0.1/LinkedTransferQueue.java:1294) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1069) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][generic][T#9]" #102 [437187] daemon prio=5 os_prio=0 cpu=68.67ms elapsed=1203.71s tid=0x00007fa18c08d570 nid=437187 waiting on condition [0x00007fa10baf9000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000eca436c0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@20.0.1/LinkedTransferQueue.java:676) - at java.util.concurrent.LinkedTransferQueue.xfer(java.base@20.0.1/LinkedTransferQueue.java:616) - at java.util.concurrent.LinkedTransferQueue.poll(java.base@20.0.1/LinkedTransferQueue.java:1294) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1069) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][generic][T#12]" #105 [437190] daemon prio=5 os_prio=0 cpu=61.65ms elapsed=1203.71s tid=0x00007fa18c090980 nid=437190 waiting on condition [0x00007fa10b7f6000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000eca436c0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@20.0.1/LinkedTransferQueue.java:676) - at java.util.concurrent.LinkedTransferQueue.xfer(java.base@20.0.1/LinkedTransferQueue.java:616) - at java.util.concurrent.LinkedTransferQueue.poll(java.base@20.0.1/LinkedTransferQueue.java:1294) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1069) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][generic][T#13]" #106 [437191] daemon prio=5 os_prio=0 cpu=51.63ms elapsed=1203.71s tid=0x00007fa18c0919e0 nid=437191 waiting on condition [0x00007fa10b6f5000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000eca436c0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@20.0.1/LinkedTransferQueue.java:676) - at java.util.concurrent.LinkedTransferQueue.xfer(java.base@20.0.1/LinkedTransferQueue.java:616) - at java.util.concurrent.LinkedTransferQueue.poll(java.base@20.0.1/LinkedTransferQueue.java:1294) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1069) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][generic][T#14]" #107 [437192] daemon prio=5 os_prio=0 cpu=66.62ms elapsed=1203.70s tid=0x00007fa1958fcc90 nid=437192 waiting on condition [0x00007fa10b5f4000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000eca436c0> (a org.elasticsearch.common.util.concurrent.EsExecutors$ExecutorScalingQueue) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.LinkedTransferQueue.awaitMatch(java.base@20.0.1/LinkedTransferQueue.java:676) - at java.util.concurrent.LinkedTransferQueue.xfer(java.base@20.0.1/LinkedTransferQueue.java:616) - at java.util.concurrent.LinkedTransferQueue.poll(java.base@20.0.1/LinkedTransferQueue.java:1294) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1069) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"elasticsearch[node_s_0][trigger_engine_scheduler][T#1]" #108 [437197] daemon prio=5 os_prio=0 cpu=0.54ms elapsed=1203.17s tid=0x00007fa18c0f4870 nid=437197 waiting on condition [0x00007fa10b0ef000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000ec9438f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@20.0.1/AbstractQueuedSynchronizer.java:1758) - at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@20.0.1/ScheduledThreadPoolExecutor.java:1182) - at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@20.0.1/ScheduledThreadPoolExecutor.java:899) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1070) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"Attach Listener" #109 [437479] daemon prio=9 os_prio=0 cpu=78.53ms elapsed=1061.49s tid=0x00007fa23c000e70 nid=437479 waiting on condition [0x0000000000000000] - java.lang.Thread.State: RUNNABLE - -"RMI TCP Accept-0" #110 [437485] daemon prio=9 os_prio=0 cpu=3.79ms elapsed=1061.30s tid=0x00007fa1880777c0 nid=437485 runnable [0x00007fa20b2e6000] - java.lang.Thread.State: RUNNABLE - at sun.nio.ch.Net.accept(java.base@20.0.1/Native Method) - at sun.nio.ch.NioSocketImpl.accept(java.base@20.0.1/NioSocketImpl.java:741) - at java.net.ServerSocket.implAccept(java.base@20.0.1/ServerSocket.java:690) - at java.net.ServerSocket.platformImplAccept(java.base@20.0.1/ServerSocket.java:655) - at java.net.ServerSocket.implAccept(java.base@20.0.1/ServerSocket.java:631) - at java.net.ServerSocket.implAccept(java.base@20.0.1/ServerSocket.java:588) - at java.net.ServerSocket.accept(java.base@20.0.1/ServerSocket.java:546) - at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(jdk.management.agent@20.0.1/LocalRMIServerSocketFactory.java:52) - at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(java.rmi@20.0.1/TCPTransport.java:424) - at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(java.rmi@20.0.1/TCPTransport.java:388) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"RMI Scheduler(0)" #112 [437488] daemon prio=9 os_prio=0 cpu=0.83ms elapsed=1061.26s tid=0x00007fa1c8070050 nid=437488 waiting on condition [0x00007fa20aec5000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000ec886be0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(java.base@20.0.1/AbstractQueuedSynchronizer.java:1758) - at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@20.0.1/ScheduledThreadPoolExecutor.java:1182) - at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(java.base@20.0.1/ScheduledThreadPoolExecutor.java:899) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1070) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"RMI TCP Connection(idle)" #126 [438704] daemon prio=9 os_prio=0 cpu=6.71ms elapsed=69.42s tid=0x00007fa2202ef430 nid=438704 waiting on condition [0x00007fa20b1e5000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000eb13d750> (a java.util.concurrent.SynchronousQueue$TransferStack) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@20.0.1/SynchronousQueue.java:401) - at java.util.concurrent.SynchronousQueue.poll(java.base@20.0.1/SynchronousQueue.java:903) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1069) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"RMI TCP Connection(idle)" #128 [438714] daemon prio=9 os_prio=0 cpu=8.77ms elapsed=61.76s tid=0x00007fa220001080 nid=438714 waiting on condition [0x00007fa10b9f8000] - java.lang.Thread.State: TIMED_WAITING (parking) - at jdk.internal.misc.Unsafe.park(java.base@20.0.1/Native Method) - - parking to wait for <0x00000000eb13d750> (a java.util.concurrent.SynchronousQueue$TransferStack) - at java.util.concurrent.locks.LockSupport.parkNanos(java.base@20.0.1/LockSupport.java:269) - at java.util.concurrent.SynchronousQueue$TransferStack.transfer(java.base@20.0.1/SynchronousQueue.java:401) - at java.util.concurrent.SynchronousQueue.poll(java.base@20.0.1/SynchronousQueue.java:903) - at java.util.concurrent.ThreadPoolExecutor.getTask(java.base@20.0.1/ThreadPoolExecutor.java:1069) - at java.util.concurrent.ThreadPoolExecutor.runWorker(java.base@20.0.1/ThreadPoolExecutor.java:1130) - at java.util.concurrent.ThreadPoolExecutor$Worker.run(java.base@20.0.1/ThreadPoolExecutor.java:642) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"JMX server connection timeout 129" #129 [438715] daemon prio=9 os_prio=0 cpu=0.28ms elapsed=61.76s tid=0x00007fa2181ddaa0 nid=438715 in Object.wait() [0x00007fa20857e000] - java.lang.Thread.State: TIMED_WAITING (on object monitor) - at java.lang.Object.wait0(java.base@20.0.1/Native Method) - - waiting on <0x00000000f93f3100> (a [I) - at java.lang.Object.wait(java.base@20.0.1/Object.java:366) - at com.sun.jmx.remote.internal.ServerCommunicatorAdmin$Timeout.run(java.management@20.0.1/ServerCommunicatorAdmin.java:171) - - locked <0x00000000f93f3100> (a [I) - at java.lang.Thread.runWith(java.base@20.0.1/Thread.java:1636) - at java.lang.Thread.run(java.base@20.0.1/Thread.java:1623) - -"VM Thread" os_prio=0 cpu=81.33ms elapsed=1208.65s tid=0x00007fa288302b80 nid=437086 runnable - -"GC Thread#0" os_prio=0 cpu=32.00ms elapsed=1208.66s tid=0x00007fa28806c5b0 nid=437081 runnable - -"GC Thread#1" os_prio=0 cpu=31.68ms elapsed=1208.29s tid=0x00007fa24c010250 nid=437109 runnable - -"GC Thread#2" os_prio=0 cpu=65.21ms elapsed=1208.29s tid=0x00007fa24c010c90 nid=437110 runnable - -"GC Thread#3" os_prio=0 cpu=54.79ms elapsed=1208.29s tid=0x00007fa24c0116d0 nid=437111 runnable - -"GC Thread#4" os_prio=0 cpu=31.41ms elapsed=1208.29s tid=0x00007fa24c012110 nid=437112 runnable - -"GC Thread#5" os_prio=0 cpu=74.60ms elapsed=1208.29s tid=0x00007fa24c012b50 nid=437113 runnable - -"GC Thread#6" os_prio=0 cpu=38.08ms elapsed=1208.29s tid=0x00007fa24c013590 nid=437114 runnable - -"GC Thread#7" os_prio=0 cpu=35.58ms elapsed=1208.29s tid=0x00007fa24c013fd0 nid=437115 runnable - -"GC Thread#8" os_prio=0 cpu=27.38ms elapsed=1208.29s tid=0x00007fa24c014a10 nid=437116 runnable - -"GC Thread#9" os_prio=0 cpu=40.27ms elapsed=1208.29s tid=0x00007fa24c015840 nid=437117 runnable - -"GC Thread#10" os_prio=0 cpu=28.91ms elapsed=1208.29s tid=0x00007fa24c0166b0 nid=437118 runnable - -"GC Thread#11" os_prio=0 cpu=22.65ms elapsed=1208.28s tid=0x00007fa24c017520 nid=437119 runnable - -"GC Thread#12" os_prio=0 cpu=28.41ms elapsed=1206.62s tid=0x00007fa24c025ca0 nid=437147 runnable - -"GC Thread#13" os_prio=0 cpu=35.51ms elapsed=1206.62s tid=0x00007fa24c026270 nid=437148 runnable - -"GC Thread#14" os_prio=0 cpu=50.15ms elapsed=1206.62s tid=0x00007fa24c026a80 nid=437149 runnable - -"GC Thread#15" os_prio=0 cpu=45.13ms elapsed=1206.62s tid=0x00007fa24c0273e0 nid=437150 runnable - -"GC Thread#16" os_prio=0 cpu=40.81ms elapsed=1206.62s tid=0x00007fa24c027d80 nid=437151 runnable - -"GC Thread#17" os_prio=0 cpu=35.62ms elapsed=1206.62s tid=0x00007fa24c0287b0 nid=437152 runnable - -"GC Thread#18" os_prio=0 cpu=19.64ms elapsed=1206.62s tid=0x00007fa24c029230 nid=437153 runnable - -"GC Thread#19" os_prio=0 cpu=15.29ms elapsed=1206.62s tid=0x00007fa24c029cb0 nid=437154 runnable - -"GC Thread#20" os_prio=0 cpu=47.43ms elapsed=1206.62s tid=0x00007fa24c02a730 nid=437155 runnable - -"GC Thread#21" os_prio=0 cpu=30.60ms elapsed=1205.14s tid=0x00007fa238179230 nid=437160 runnable - -"GC Thread#22" os_prio=0 cpu=22.25ms elapsed=1205.14s tid=0x00007fa20c116ff0 nid=437161 runnable - -"GC Thread#23" os_prio=0 cpu=2.17ms elapsed=1205.14s tid=0x00007fa24c0311e0 nid=437162 runnable - -"GC Thread#24" os_prio=0 cpu=12.68ms elapsed=1205.14s tid=0x00007fa24c031640 nid=437163 runnable - -"GC Thread#25" os_prio=0 cpu=28.13ms elapsed=1205.14s tid=0x00007fa24c031c20 nid=437164 runnable - -"GC Thread#26" os_prio=0 cpu=2.24ms elapsed=1205.14s tid=0x00007fa24c032330 nid=437165 runnable - -"GC Thread#27" os_prio=0 cpu=18.80ms elapsed=1205.14s tid=0x00007fa24c032a60 nid=437166 runnable - -"GC Thread#28" os_prio=0 cpu=30.58ms elapsed=1205.14s tid=0x00007fa24c033250 nid=437167 runnable - -"GC Thread#29" os_prio=0 cpu=3.36ms elapsed=1205.14s tid=0x00007fa24c033b10 nid=437168 runnable - -"GC Thread#30" os_prio=0 cpu=10.35ms elapsed=1205.14s tid=0x00007fa24c0343d0 nid=437169 runnable - -"GC Thread#31" os_prio=0 cpu=10.74ms elapsed=1205.14s tid=0x00007fa24c034e50 nid=437170 runnable - -"GC Thread#32" os_prio=0 cpu=0.40ms elapsed=756.55s tid=0x00007fa2300f25e0 nid=437889 runnable - -"GC Thread#33" os_prio=0 cpu=0.43ms elapsed=756.55s tid=0x00007fa214076f60 nid=437890 runnable - -"GC Thread#34" os_prio=0 cpu=0.45ms elapsed=756.55s tid=0x00007fa20c1be560 nid=437891 runnable - -"GC Thread#35" os_prio=0 cpu=0.59ms elapsed=756.55s tid=0x00007fa24c02ece0 nid=437892 runnable - -"GC Thread#36" os_prio=0 cpu=0.56ms elapsed=756.55s tid=0x00007fa24c030930 nid=437893 runnable - -"GC Thread#37" os_prio=0 cpu=0.36ms elapsed=756.55s tid=0x00007fa24c0372e0 nid=437894 runnable - -"GC Thread#38" os_prio=0 cpu=0.29ms elapsed=756.55s tid=0x00007fa24c037740 nid=437895 runnable - -"GC Thread#39" os_prio=0 cpu=0.48ms elapsed=756.55s tid=0x00007fa24c037e40 nid=437896 runnable - -"GC Thread#40" os_prio=0 cpu=0.32ms elapsed=756.55s tid=0x00007fa24c038610 nid=437897 runnable - -"GC Thread#41" os_prio=0 cpu=0.17ms elapsed=756.55s tid=0x00007fa24c0391d0 nid=437898 runnable - -"GC Thread#42" os_prio=0 cpu=0.60ms elapsed=756.55s tid=0x00007fa24c039d90 nid=437899 runnable - -"G1 Main Marker" os_prio=0 cpu=1.54ms elapsed=1208.66s tid=0x00007fa288071ca0 nid=437082 runnable - -"G1 Conc#0" os_prio=0 cpu=10.22ms elapsed=1208.66s tid=0x00007fa288072c00 nid=437083 runnable - -"G1 Conc#1" os_prio=0 cpu=8.42ms elapsed=1207.47s tid=0x00007fa260000c10 nid=437128 runnable - -"G1 Conc#2" os_prio=0 cpu=12.18ms elapsed=1207.47s tid=0x00007fa260001660 nid=437129 runnable - -"G1 Conc#3" os_prio=0 cpu=11.61ms elapsed=1207.47s tid=0x00007fa2600020e0 nid=437130 runnable - -"G1 Conc#4" os_prio=0 cpu=10.70ms elapsed=1207.47s tid=0x00007fa260002b60 nid=437131 runnable - -"G1 Conc#5" os_prio=0 cpu=10.55ms elapsed=1207.47s tid=0x00007fa2600035e0 nid=437132 runnable - -"G1 Conc#6" os_prio=0 cpu=7.83ms elapsed=1207.47s tid=0x00007fa260004060 nid=437133 runnable - -"G1 Conc#7" os_prio=0 cpu=9.93ms elapsed=1207.47s tid=0x00007fa260004ae0 nid=437134 runnable - -"G1 Conc#8" os_prio=0 cpu=9.89ms elapsed=1207.46s tid=0x00007fa260005560 nid=437135 runnable - -"G1 Conc#9" os_prio=0 cpu=10.07ms elapsed=1207.46s tid=0x00007fa260005fe0 nid=437136 runnable - -"G1 Conc#10" os_prio=0 cpu=8.98ms elapsed=1207.46s tid=0x00007fa260006a60 nid=437137 runnable - -"G1 Refine#0" os_prio=0 cpu=0.08ms elapsed=1208.65s tid=0x00007fa2882da2b0 nid=437084 runnable - -"G1 Service" os_prio=0 cpu=21.00ms elapsed=1208.65s tid=0x00007fa2882db1f0 nid=437085 runnable - -"VM Periodic Task Thread" os_prio=0 cpu=334.35ms elapsed=1208.62s tid=0x00007fa28836a730 nid=437097 waiting on condition - -JNI global refs: 48, weak refs: 45 - diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/RotatableSecret.java b/server/src/main/java/org/elasticsearch/common/settings/RotatableSecret.java similarity index 78% rename from x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/RotatableSecret.java rename to server/src/main/java/org/elasticsearch/common/settings/RotatableSecret.java index c9c7ebbda9392..e2bdd3b5e20a2 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/RotatableSecret.java +++ b/server/src/main/java/org/elasticsearch/common/settings/RotatableSecret.java @@ -1,14 +1,13 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ - -package org.elasticsearch.xpack.security.support; +package org.elasticsearch.common.settings; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; @@ -16,9 +15,9 @@ import java.util.concurrent.locks.StampedLock; /** - * Helper class to provide a secret that can be rotated. Once rotated the prior secret is available for a configured amount of time before - * it is invalidated. This allows for secrete rotation without temporary failures or the need to tightly orchestrate multiple parties. - * This class is threadsafe, however it is also assumes that matching secrets are frequent but rotation is a rare. + * Helper class to provide {@link SecureString} that can be rotated. Once rotated the prior secret is available for a configured amount + * of time before it is invalidated. This allows for secrete rotation without temporary failures or the need to tightly orchestrate + * multiple parties. This class is threadsafe, however it is also assumes that matching secrets are frequent but rotation is a rare. */ public class RotatableSecret { private Secrets secrets; @@ -28,7 +27,7 @@ public class RotatableSecret { * @param secret The secret to rotate. {@code null} if the secret is not configured. */ public RotatableSecret(@Nullable SecureString secret) { - this.secrets = new Secrets(Strings.hasText(secret) ? secret : null, null, Instant.EPOCH); + this.secrets = new Secrets(Strings.hasText(secret) ? secret.clone() : null, null, Instant.EPOCH); } /** @@ -42,7 +41,7 @@ public void rotate(SecureString newSecret, TimeValue gracePeriod) { try { if (this.secrets.current.equals(newSecret) == false) { secrets = new Secrets( - Strings.hasText(newSecret) ? newSecret : null, + Strings.hasText(newSecret) ? newSecret.clone() : null, secrets.current, Instant.now().plusMillis(gracePeriod.getMillis()) ); @@ -74,11 +73,20 @@ public boolean matches(SecureString secret) { return secrets.current.equals(secret) || (secrets.prior != null && secrets.prior.equals(secret)); } - // for testing purpose only + // for testing only + //TODO: adjust test so that i don't need to make this public public Secrets getSecrets() { return secrets; } + // for testing only + boolean isWriteLocked() { + return stampedLock.isWriteLocked(); + } + + /** + * Checks to see if the prior secret TTL has expired. If expired, evict from the backing data structure. + */ private void checkExpired() { boolean needToUnlock = false; long stamp = stampedLock.tryOptimisticRead(); @@ -91,7 +99,7 @@ private void checkExpired() { } try { if (expired) { - stamp = stampedLock.tryConvertToWriteLock(stamp);// upgrade the read lock + stamp = stampedLock.tryConvertToWriteLock(stamp); if (stamp == 0) { // block until we can acquire the write lock stamp = stampedLock.writeLock(); diff --git a/server/src/test/java/org/elasticsearch/common/settings/RotatableSecretTests.java b/server/src/test/java/org/elasticsearch/common/settings/RotatableSecretTests.java new file mode 100644 index 0000000000000..1d4dadd0a5e70 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/settings/RotatableSecretTests.java @@ -0,0 +1,220 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.settings; + +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESTestCase; +import org.mockito.stubbing.Answer; + +import java.time.Instant; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RotatableSecretTests extends ESTestCase { + + private final SecureString secret1 = new SecureString(randomAlphaOfLength(10)); + private final SecureString secret2 = new SecureString(randomAlphaOfLength(10)); + private final SecureString secret3 = new SecureString(randomAlphaOfLength(10)); + + public void testBasicRotation() throws Exception { + // initial state + RotatableSecret rotatableSecret = new RotatableSecret(secret1); + assertTrue(rotatableSecret.matches(secret1)); + assertFalse(rotatableSecret.matches(secret2)); + assertFalse(rotatableSecret.matches(new SecureString(randomAlphaOfLength(10)))); + assertTrue(rotatableSecret.isSet()); + assertEquals(secret1, rotatableSecret.getSecrets().current()); + assertNull(rotatableSecret.getSecrets().prior()); + assertEquals(Instant.EPOCH, rotatableSecret.getSecrets().priorValidTill()); + + // normal rotation + TimeValue expiresIn = TimeValue.timeValueDays(1); + rotatableSecret.rotate(secret2, expiresIn); + assertTrue(rotatableSecret.matches(secret1)); + assertTrue(rotatableSecret.matches(secret2)); + assertFalse(rotatableSecret.matches(new SecureString(randomAlphaOfLength(10)))); + assertTrue(rotatableSecret.isSet()); + assertEquals(secret2, rotatableSecret.getSecrets().current()); + assertEquals(secret1, rotatableSecret.getSecrets().prior()); + assertTrue(rotatableSecret.getSecrets().priorValidTill().isAfter(Instant.now())); + assertTrue( + rotatableSecret.getSecrets().priorValidTill().isBefore(Instant.now().plusMillis(TimeValue.timeValueDays(2).getMillis())) + ); + + // attempt to rotate same value does nothing + rotatableSecret.rotate(secret2, TimeValue.timeValueDays(99)); // ignores the new expiry since you can't rotate the same secret + assertTrue(rotatableSecret.matches(secret1)); + assertTrue(rotatableSecret.matches(secret2)); + assertFalse(rotatableSecret.matches(new SecureString(randomAlphaOfLength(10)))); + assertTrue(rotatableSecret.isSet()); + assertEquals(secret2, rotatableSecret.getSecrets().current()); + assertEquals(secret1, rotatableSecret.getSecrets().prior()); + assertTrue(rotatableSecret.getSecrets().priorValidTill().isAfter(Instant.now())); + assertTrue( + rotatableSecret.getSecrets().priorValidTill().isBefore(Instant.now().plusMillis(TimeValue.timeValueDays(2).getMillis())) + ); + + // rotate with expiry + rotatableSecret.rotate(secret3, TimeValue.timeValueMillis(1)); + Thread.sleep(2); // ensure secret2 is expired + assertTrue(rotatableSecret.matches(secret3)); + assertFalse(rotatableSecret.matches(secret1)); + assertFalse(rotatableSecret.matches(secret2)); + assertFalse(rotatableSecret.matches(new SecureString(randomAlphaOfLength(10)))); + assertTrue(rotatableSecret.isSet()); + assertEquals(secret3, rotatableSecret.getSecrets().current()); + assertNull(rotatableSecret.getSecrets().prior()); + assertTrue(rotatableSecret.getSecrets().priorValidTill().isBefore(Instant.now())); + + // unset current and prior + rotatableSecret.rotate(null, TimeValue.ZERO); + assertFalse(rotatableSecret.matches(secret3)); + assertFalse(rotatableSecret.matches(secret1)); + assertFalse(rotatableSecret.matches(secret2)); + assertFalse(rotatableSecret.matches(new SecureString(randomAlphaOfLength(10)))); + assertFalse(rotatableSecret.isSet()); + assertNull(rotatableSecret.getSecrets().current()); + assertNull(rotatableSecret.getSecrets().prior()); + assertTrue(rotatableSecret.getSecrets().priorValidTill().isBefore(Instant.now())); + } + + public void testConcurrentReadWhileLocked() throws Exception { + // initial state + RotatableSecret rotatableSecret = new RotatableSecret(secret1); + assertTrue(rotatableSecret.matches(secret1)); + assertFalse(rotatableSecret.matches(secret2)); + assertEquals(secret1, rotatableSecret.getSecrets().current()); + assertNull(rotatableSecret.getSecrets().prior()); + + CountDownLatch latch = new CountDownLatch(1); + TimeValue mockGracePeriod = mock(TimeValue.class); // use a mock to force a long rotation to exercise the concurrency + when(mockGracePeriod.getMillis()).then((Answer) invocation -> { + latch.await(); + return Long.MAX_VALUE; + }); + + // start writer thread + Thread t1 = new Thread(() -> rotatableSecret.rotate(secret2, mockGracePeriod)); + t1.start(); + assertBusy(() -> assertEquals(Thread.State.WAITING, t1.getState())); // waiting on countdown latch, holds write lock + + // start reader threads + int readers = randomIntBetween(0, 16); + Set readerThreads = new HashSet<>(readers); + for (int i = 0; i <= readers; i++) { + Thread t = new Thread(() -> { + if (randomBoolean()) { // either matches or isSet can block + assertTrue(rotatableSecret.matches(secret1)); + assertTrue(rotatableSecret.matches(secret2)); + } else { + assertTrue(rotatableSecret.isSet()); + } + }); + readerThreads.add(t); + t.start(); + } + for (Thread t : readerThreads) { + assertBusy(() -> assertEquals(Thread.State.WAITING, t.getState())); // waiting on write lock from thread 1 to be released + } + assertTrue(rotatableSecret.isWriteLocked()); + latch.countDown(); // let thread1 finish, which also unblocks the reader threads + assertBusy(() -> assertEquals(Thread.State.TERMINATED, t1.getState())); // done with work + assertFalse(rotatableSecret.isWriteLocked()); + for (Thread t : readerThreads) { + assertBusy(() -> assertEquals(Thread.State.TERMINATED, t.getState())); // done with work + t.join(); + } + t1.join(); + } + + public void testConcurrentRotations() throws Exception { + // initial state + RotatableSecret rotatableSecret = new RotatableSecret(secret1); + assertTrue(rotatableSecret.matches(secret1)); + assertFalse(rotatableSecret.matches(secret2)); + assertEquals(secret1, rotatableSecret.getSecrets().current()); + assertNull(rotatableSecret.getSecrets().prior()); + + // start first rotation + AtomicBoolean latch1 = new AtomicBoolean(false); // using boolean as latch to differentiate the kinds of waiting + TimeValue mockGracePeriod1 = mock(TimeValue.class); // use a mock to force a long rotation to exercise the concurrency + when(mockGracePeriod1.getMillis()).then((Answer) invocation -> { + while (latch1.get() == false) { + Thread.sleep(10); // thread in TIMED_WAITING + } + return Long.MAX_VALUE; + }); + Thread t1 = new Thread(() -> rotatableSecret.rotate(secret2, mockGracePeriod1)); + t1.start(); + assertBusy(() -> assertEquals(Thread.State.TIMED_WAITING, t1.getState())); // waiting on latch, holds write lock + + // start second rotation + AtomicBoolean latch2 = new AtomicBoolean(false); + TimeValue mockGracePeriod2 = mock(TimeValue.class); // use a mock to force a long rotation to exercise the concurrency + when(mockGracePeriod2.getMillis()).then((Answer) invocation -> { + while (latch2.get() == false) { + Thread.sleep(10); // thread in TIMED_WAITING + } + return Long.MAX_VALUE; + }); + Thread t2 = new Thread(() -> rotatableSecret.rotate(secret3, mockGracePeriod2)); + t2.start(); + assertBusy(() -> assertEquals(Thread.State.WAITING, t2.getState())); // waiting on write lock from thread 1 + + // start third rotation + AtomicBoolean latch3 = new AtomicBoolean(false); + TimeValue mockGracePeriod3 = mock(TimeValue.class); // use a mock to force a long rotation to exercise the concurrency + when(mockGracePeriod3.getMillis()).then((Answer) invocation -> { + while (latch3.get() == false) { + Thread.sleep(10); // thread in TIMED_WAITING + } + return Long.MAX_VALUE; + }); + Thread t3 = new Thread(() -> rotatableSecret.rotate(null, mockGracePeriod3)); + t3.start(); + assertBusy(() -> assertEquals(Thread.State.WAITING, t3.getState())); // waiting on write lock from thread 1 + + // initial state + assertEquals(rotatableSecret.getSecrets().current(), secret1); + assertNull(rotatableSecret.getSecrets().prior()); + assertBusy(() -> assertEquals(Thread.State.TIMED_WAITING, t1.getState())); // waiting on latch + assertBusy(() -> assertEquals(Thread.State.WAITING, t2.getState())); // waiting on lock + assertBusy(() -> assertEquals(Thread.State.WAITING, t3.getState())); // waiting on lock + + latch1.set(true); // let first rotation succeed + assertBusy(() -> assertEquals(Thread.State.TERMINATED, t1.getState())); // work done + assertBusy(() -> assertEquals(Thread.State.TIMED_WAITING, t2.getState())); // waiting on latch + assertBusy(() -> assertEquals(Thread.State.WAITING, t3.getState())); // waiting lock + assertEquals(rotatableSecret.getSecrets().current(), secret2); + assertEquals(rotatableSecret.getSecrets().prior(), secret1); + + latch2.set(true); // let second rotation succeed + assertBusy(() -> assertEquals(Thread.State.TERMINATED, t1.getState())); // work done + assertBusy(() -> assertEquals(Thread.State.TERMINATED, t2.getState())); // work done + assertBusy(() -> assertEquals(Thread.State.TIMED_WAITING, t3.getState())); // waiting on latch + assertEquals(rotatableSecret.getSecrets().current(), secret3); + assertEquals(rotatableSecret.getSecrets().prior(), secret2); + + latch3.set(true); // let third rotation succeed + assertBusy(() -> assertEquals(Thread.State.TERMINATED, t1.getState())); // work done + assertBusy(() -> assertEquals(Thread.State.TERMINATED, t2.getState())); // work done + assertBusy(() -> assertEquals(Thread.State.TERMINATED, t3.getState())); // work done + assertEquals(rotatableSecret.getSecrets().current(), null); + assertEquals(rotatableSecret.getSecrets().prior(), secret3); + + t1.join(); + t2.join(); + t3.join(); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmSettings.java index b55973d192389..b09e3cfe07e14 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/jwt/JwtRealmSettings.java @@ -352,13 +352,13 @@ public Iterator> settings() { ) ); - //TODO: document that this is reloadable + // TODO: document that this is reloadable public static final Setting.AffixSetting CLIENT_AUTHENTICATION_SHARED_SECRET = RealmSettings.secureString( TYPE, "client_authentication.shared_secret" ); - //TODO: document this ! + // TODO: document this ! public static final Setting.AffixSetting CLIENT_AUTHENTICATION_SHARED_SECRET_ROTATION_GRACE = Setting.affixKeySetting( RealmSettings.realmSettingPrefix(TYPE), "client_authentication.rotation_grace_period", diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java index 6e5b99ea6e85f..d70ff96369169 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java @@ -198,26 +198,32 @@ public void testClientSecretRotation() throws Exception { String jwt0SharedSecret = "jwt0_shared_secret"; String jwt1SharedSecret = "jwt1_shared_secret"; String jwt2SharedSecret = "jwt2_shared_secret"; - //sanity check + // sanity check assertThat(getCurrentSecret(realm0), equalTo(jwt0SharedSecret)); assertThat(getCurrentSecret(realm1), equalTo(jwt1SharedSecret)); assertThat(getCurrentSecret(realm2), equalTo(jwt2SharedSecret)); final String update1suffix = "_update1"; final MockSecureSettings newSecureSettings = new MockSecureSettings(); - newSecureSettings.setString("xpack.security.authc.realms.jwt." + realm0.name() + ".client_authentication.shared_secret", jwt0SharedSecret + update1suffix); - newSecureSettings.setString("xpack.security.authc.realms.jwt." + realm1.name() + ".client_authentication.shared_secret", jwt1SharedSecret + update1suffix); - newSecureSettings.setString("xpack.security.authc.realms.jwt." + realm2.name() + ".client_authentication.shared_secret", jwt2SharedSecret + update1suffix); - - + newSecureSettings.setString( + "xpack.security.authc.realms.jwt." + realm0.name() + ".client_authentication.shared_secret", + jwt0SharedSecret + update1suffix + ); + newSecureSettings.setString( + "xpack.security.authc.realms.jwt." + realm1.name() + ".client_authentication.shared_secret", + jwt1SharedSecret + update1suffix + ); + newSecureSettings.setString( + "xpack.security.authc.realms.jwt." + realm2.name() + ".client_authentication.shared_secret", + jwt2SharedSecret + update1suffix + ); final PluginsService plugins = getInstanceFromNode(PluginsService.class); final LocalStateSecurity notTheDroids = plugins.filterPlugins(LocalStateSecurity.class).get(0); for (Plugin p : notTheDroids.plugins()) { if (p instanceof Security securityPlugin) { - Settings.Builder newSettingsBuilder = Settings.builder() - .setSecureSettings(newSecureSettings); + Settings.Builder newSettingsBuilder = Settings.builder().setSecureSettings(newSecureSettings); securityPlugin.reload(newSettingsBuilder.build()); } } @@ -227,36 +233,36 @@ public void testClientSecretRotation() throws Exception { assertTrue(realm0.getClientAuthenticationSharedSecret().matches(new SecureString(jwt0SharedSecret))); assertTrue(realm0.getClientAuthenticationSharedSecret().matches(new SecureString(jwt0SharedSecret + update1suffix))); - assertTrue("possible timing issue with test", Instant.now().isBefore(realm1.getClientAuthenticationSharedSecret().getSecrets().priorValidTill())); - assertTrue(realm1.getClientAuthenticationSharedSecret().matches(new SecureString(jwt1SharedSecret))); //TODO: safegaurd this against slow executions assert above this + assertTrue( + "possible timing issue with test", + Instant.now().isBefore(realm1.getClientAuthenticationSharedSecret().getSecrets().priorValidTill()) + ); + assertTrue(realm1.getClientAuthenticationSharedSecret().matches(new SecureString(jwt1SharedSecret))); // TODO: safegaurd this + // against slow executions + // assert above this assertTrue(realm1.getClientAuthenticationSharedSecret().matches(new SecureString(jwt1SharedSecret + update1suffix))); - assertFalse(realm2.getClientAuthenticationSharedSecret().matches(new SecureString(jwt2SharedSecret))); //disabled old + assertFalse(realm2.getClientAuthenticationSharedSecret().matches(new SecureString(jwt2SharedSecret))); // disabled old assertTrue(realm2.getClientAuthenticationSharedSecret().matches(new SecureString(jwt2SharedSecret + update1suffix))); - Thread.sleep(1000); - assertTrue("possible timing issue with test", Instant.now().isAfter(realm1.getClientAuthenticationSharedSecret().getSecrets().priorValidTill())); - assertFalse(realm1.getClientAuthenticationSharedSecret().matches(new SecureString(jwt2SharedSecret))); //old expired - - - - - - + Thread.sleep(1000); //TODO: convert this to busy wait + assertTrue( + "possible timing issue with test", + Instant.now().isAfter(realm1.getClientAuthenticationSharedSecret().getSecrets().priorValidTill()) + ); + assertFalse(realm1.getClientAuthenticationSharedSecret().matches(new SecureString(jwt2SharedSecret))); // old expired } - private SecureString getPriorSecret(JwtRealm realm){ + private SecureString getPriorSecret(JwtRealm realm) { return realm.getClientAuthenticationSharedSecret().getSecrets().prior(); } - private SecureString getCurrentSecret(JwtRealm realm){ + private SecureString getCurrentSecret(JwtRealm realm) { return realm.getClientAuthenticationSharedSecret().getSecrets().current(); } - private TimeValue getGracePeriod(JwtRealm realm){ - return realm.getConfig() - .getConcreteSetting(CLIENT_AUTHENTICATION_SHARED_SECRET_ROTATION_GRACE) - .get(realm.getConfig().settings()); + private TimeValue getGracePeriod(JwtRealm realm) { + return realm.getConfig().getConcreteSetting(CLIENT_AUTHENTICATION_SHARED_SECRET_ROTATION_GRACE).get(realm.getConfig().settings()); } private void assertJwtToken(JwtAuthenticationToken token, String tokenPrincipal, String sharedSecret, SignedJWT signedJWT) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java index d6dc7c46c22ce..5c04da1b3bc9b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealm.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.cache.Cache; import org.elasticsearch.common.cache.CacheBuilder; +import org.elasticsearch.common.settings.RotatableSecret; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.util.concurrent.ReleasableLock; @@ -34,7 +35,6 @@ import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.security.authc.support.ClaimParser; import org.elasticsearch.xpack.security.authc.support.DelegatedAuthorizationSupport; -import org.elasticsearch.xpack.security.support.RotatableSecret; import java.text.ParseException; import java.util.ArrayList; @@ -448,7 +448,7 @@ public void rotateClientSecret(SecureString clientSecret) { this.clientAuthenticationSharedSecret.rotate(clientSecret, config.getSetting(CLIENT_AUTHENTICATION_SHARED_SECRET_ROTATION_GRACE)); } - //package private for testing + // package private for testing RotatableSecret getClientAuthenticationSharedSecret() { return clientAuthenticationSharedSecret; } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java index 65d335d91bb28..3e3533f028b38 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtil.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.SuppressLoggerChecks; import org.elasticsearch.common.hash.MessageDigests; +import org.elasticsearch.common.settings.RotatableSecret; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.common.ssl.SslConfiguration; @@ -44,7 +45,6 @@ import org.elasticsearch.xpack.core.security.authc.RealmSettings; import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings; import org.elasticsearch.xpack.core.ssl.SSLService; -import org.elasticsearch.xpack.security.support.RotatableSecret; import java.io.InputStream; import java.net.URI; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtilTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtilTests.java index ca9f002bf4fc1..6fab33b4d6adf 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtilTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/jwt/JwtUtilTests.java @@ -6,10 +6,10 @@ */ package org.elasticsearch.xpack.security.authc.jwt; +import org.elasticsearch.common.settings.RotatableSecret; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.SettingsException; import org.elasticsearch.xpack.core.security.authc.jwt.JwtRealmSettings; -import org.elasticsearch.xpack.security.support.RotatableSecret; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is;