kv/kvnemesis: TestKVNemesisMultiNode failed

[cockroach-teamcity]

kv/kvnemesis.TestKVNemesisMultiNode failed with artifacts on release-23.2 @ 77237763413fff744e598f85a8b554ac3de66c38:

Fatal error:

panic: r99/1: closed timestamp regression 1729335828.458953533,0 -> 1729335828.457872252,0 (cmd "091d7012f56c394f" index 14 term 6 (LAI=2 CTS=1729335828.457872252,0))
data: 00091d7012f56c394f300b6a12420a08b6a1a8d9ebb8f5ff1792010360b80372c6020ac3020000000000000000020000000139017a6b12016b12ec126536366230343435366538613063353400ffff0100ff017264736300010003a74ebdfa4a374cf3881fc9414925b6c5126a0a520a10a74ebdfa4a374cf3881fc9414925b6c51a1e016b12ec126536366230343435366538613063353400ff010001726473632a0a08b6a1a8d9ebb8f5ff1730c8fd0d38014a0a08b6a1a8d9ebb8f5ff175002120c08b6a1a8d9ebb8f5ff1710001800200c286648010128016b12ec126536366230343435366538613063353400ff010001726473630017ffd5c6bb2a10b6096600000002650a00c5b033130308631214ec126536366230343435366538613063353400011a02ffff220808031003180120002208080110011802200022080802100218032000220808041004180420012805302e3a1008ffffffffffffffff7f10ffffffff077a008201260a12637264622d7472616365722d7370616e69641210323863626637333036326264646361378201270a13637264622d7472616365722d747261636569641210316564356566656238613165666637648201080a0372656312017620028a010a08fcb6d9c4e0b8f5ff17 [recovered]
	panic: r99/1: closed timestamp regression 1729335828.458953533,0 -> 1729335828.457872252,0 (cmd "091d7012f56c394f" index 14 term 6 (LAI=2 CTS=1729335828.457872252,0))
data: 00091d7012f56c394f300b6a12420a08b6a1a8d9ebb8f5ff1792010360b80372c6020ac3020000000000000000020000000139017a6b12016b12ec126536366230343435366538613063353400ffff0100ff017264736300010003a74ebdfa4a374cf3881fc9414925b6c5126a0a520a10a74ebdfa4a374cf3881fc9414925b6c51a1e016b12ec126536366230343435366538613063353400ff010001726473632a0a08b6a1a8d9ebb8f5ff1730c8fd0d38014a0a08b6a1a8d9ebb8f5ff175002120c08b6a1a8d9ebb8f5ff1710001800200c286648010128016b12ec126536366230343435366538613063353400ff010001726473630017ffd5c6bb2a10b6096600000002650a00c5b033130308631214ec126536366230343435366538613063353400011a02ffff220808031003180120002208080110011802200022080802100218032000220808041004180420012805302e3a1008ffffffffffffffff7f10ffffffff077a008201260a12637264622d7472616365722d7370616e69641210323863626637333036326264646361378201270a13637264622d7472616365722d747261636569641210316564356566656238613165666637648201080a0372656312017620028a010a08fcb6d9c4e0b8f5ff17

Stack:

goroutine 29638 [running]:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0x1f00bc5?, {0x6be89f0, 0xc009fc59b0})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:225 +0x65
panic({0x46ae7a0?, 0xc00fdc62e0?})
	GOROOT/src/runtime/panic.go:914 +0x21f
github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*rangeAsserter).apply.func1({0x503635a?, 0xc00eecb1d0?}, {0xc01461f9f0?, 0x8?, 0x46bddc?})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/asserter.go:245 +0x165
github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*rangeAsserter).apply(0xc005981810, 0x1, {0xc00cf34628, 0x8}, {0x6, 0xe, 0x0, {0xc009460960, 0x1d7, 0x1e0}}, ...)
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/asserter.go:296 +0x5ff
github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Asserter).Apply(0x6be88d8?, 0x9f23840?, 0x6bab280?, {0xc00cf34628, 0x8}, {0x6, 0xe, 0x0, {0xc009460960, 0x1d7, ...}}, ...)
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/asserter.go:226 +0xa5
github.com/cockroachdb/cockroach/pkg/kv/kvnemesis.kvnemesisTestCfg.testClusterArgs.func7({{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, {0x17ffd5c6bb2a10b6, 0x0, 0x0}, {0x0, ...}, ...}, ...})
	github.com/cockroachdb/cockroach/pkg/kv/kvnemesis/pkg/kv/kvnemesis/kvnemesis_test.go:141 +0x86
github.com/cockroachdb/cockroach/pkg/kv/kvserver.replicaApplyTestingFilters({0xc00a956690?, 0x6be89f0?}, 0xc017688d80?, 0xc001b9fa28?, {0xc00a9566e0?, 0xc0052b8e10?, 0x0?}, 0xf0?)
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_application_state_machine.go:112 +0x1f7
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaAppBatch).Stage(0xc00a956690, {0x6be89f0, 0xc017688d80}, {0x6c08960?, 0xc001b9fa28})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_app_batch.go:108 +0xe9
github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.mapCmdIter({0x6c17930?, 0xc00a956a38}, 0xc008686598)
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/cmd.go:187 +0x148
github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).applyOneBatch(0xc008686af8, {0x6be89f0, 0xc00811ea80}, {0x6c17930, 0xc00a956a08})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:279 +0x150
github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply.(*Task).ApplyCommittedEntries(0xc008686af8, {0x6be89f0, 0xc00811ea80})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/apply/task.go:246 +0x95
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReadyRaftMuLocked(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:1060 +0x13c5
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleRaftReady(_, {_, _}, {{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...}, ...})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/replica_raft.go:745 +0x1b8
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).processReady(0xc001e37500, 0x6be89f0?)
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:684 +0x12b
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftSchedulerShard).worker(0xc00ae5b7a0, {0x6be89f0, 0xc009fc59b0}, {0x6bcd3c0, 0xc001e37500}, 0xc0094b3000)
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:413 +0x1e6
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2({0x6be89f0?, 0xc009fc59b0?})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/scheduler.go:316 +0x46
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2()
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:479 +0x13a
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx in goroutine 25885
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470 +0x415

Log preceding fatal error

=== RUN   TestKVNemesisMultiNode
    test_log_scope.go:165: test logs captured to: /artifacts/tmp/_tmp/1f42cf5be2fc021646bf9b2daf5eaef3/logTestKVNemesisMultiNode3369187969
    test_log_scope.go:76: use -show-logs to present logs inline
    kvnemesis_test.go:315: seed: 7488000090895233454
    kvnemesis_test.go:222: kvnemesis logging to /artifacts/tmp/_tmp/1f42cf5be2fc021646bf9b2daf5eaef3/kvnemesis3740258545

Parameters:

• TAGS=bazel,gss
• stress=true

Help

See also: How To Investigate a Go Test Failure (internal)

/cc @cockroachdb/kv

This test on roachdash | Improve this report!

Jira issue: CRDB-43383