sql: TestTenantGlobalAggregatedLivebytes failed

[cockroach-teamcity]

sql.TestTenantGlobalAggregatedLivebytes failed on release-24.3 @ 4cbedefd790c75cb0f21f77ed8d917c8528a7d15:

Fatal error:

panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x185aad1]

Stack:

goroutine 60094 [running]:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0xc011503868?, {0x8ba9ae8, 0xc00d4f2a80})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:226 +0x65
panic({0x66ccc00?, 0xce14b40?})
	GOROOT/src/runtime/panic.go:770 +0x132
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).recover(0xc011503988?, {0x8ba9ae8, 0xc00d4f2bd0})
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:226 +0x65
panic({0x66ccc00?, 0xce14b40?})
	GOROOT/src/runtime/panic.go:770 +0x132
github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotQueue).Admit(0x0, {0x8ba9ae8, 0xc00d4f2c90}, 0x562)
	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:165 +0x71
github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotPacer).Pace(0xc011504768, {0x8ba9ae8?, 0xc00d4f2c90?}, 0xc00e818580?, 0xe8?)
	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:292 +0x6d
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*kvBatchSnapshotStrategy).Receive(_, {_, _}, _, {_, _}, {{0xe, 0x2, 0xc006aa1c70, 0xc006aa1ce0, ...}, ...}, ...)
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:918 +0x1f9f
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).receiveSnapshot(0xc0095fc808, {0x8ba9ae8, 0xc00d4f2bd0}, 0xc00d4f6248, {0x7fa06131af80, 0xc0079fa8e0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:1693 +0x76b
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot.func1({0x8ba9ae8, 0xc00d4f2bd0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:209 +0x85
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc0079e4870, {0x8ba9ae8, 0xc00d4f2bd0}, {0x728fb0b, 0x1e}, 0xc00ae3b280)
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x15d
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot(0xc0095fc808, {0x8ba9ab0?, 0xc00d3ef3b0?}, 0xc00d4f6248, {0x7fa06131af58, 0xc0079fa8e0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:206 +0xf6
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftSnapshot(0xc007e20380, {0x8bf7df0, 0xc0079fa8e0})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:638 +0x32d
github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftSnapshot_Handler({0x70c0160?, 0xc007e20380}, {0x8be1300, 0xc002cd3080})
	github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-fastbuild/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:267 +0xd8
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.StreamServerInterceptor.func13({0x70c0160, 0xc007e20380}, {0x8be2538, 0xc007b8c780}, 0xc00c5d3dd0, 0x76b94a0)
	github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:160 +0x35b
google.golang.org/grpc.getChainStreamHandler.func1({0x70c0160, 0xc007e20380}, {0x8be2538, 0xc007b8c780})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4({0x70c0160, 0xc007e20380}, {0x8be2538, 0xc007b8c780}, 0xc00c5d3dd0?, 0xc00a842d00)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:174 +0x6c
google.golang.org/grpc.getChainStreamHandler.func1({0x70c0160, 0xc007e20380}, {0x8be2538, 0xc007b8c780})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor({0xc003a44000?, {{0x470445?}, {0x8c03500?, 0xc0052b3c80?}}}, {0x70c0160, 0xc007e20380}, {0x8be2538, 0xc007b8c780}, 0xc00c5d3dd0, 0xc00a842cc0)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:152 +0x384
google.golang.org/grpc.getChainStreamHandler.func1({0x70c0160, 0xc007e20380}, {0x8be2538, 0xc007b8c780})
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481 +0xb2
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1({0x7fa064a72668?, 0x8ba9ae8?})
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:141 +0x27
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc0079e4870, {0x8ba9ae8, 0xc00d4f2a80}, {0xc00947ef30, 0x29}, 0xc00a92daa0)
	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349 +0x15d
github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2({0x70c0160, 0xc007e20380}, {0x8be2538, 0xc007b8c780}, 0xc00c5d3dd0, 0xc00a842c80)
	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:140 +0xdf
google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2({0x70c0160, 0xc007e20380}, {0x8be2538, 0xc007b8c780}, 0xc00c5d3dd0, 0xc0079fa8d0?)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1472 +0x85
google.golang.org/grpc.(*Server).processStreamingRPC(0xc00565c1e0, {0x8c03e60, 0xc00cb65520}, 0xc00d3817a0, 0xc009f0f500, 0xcf11d80, 0x0)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636 +0x1247
google.golang.org/grpc.(*Server).handleStream(0xc00565c1e0, {0x8c03e60, 0xc00cb65520}, 0xc00d3817a0, 0x0)
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1716 +0x999
google.golang.org/grpc.(*Server).serveStreams.func1.1()
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:947 +0xbb
created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 60196
	google.golang.org/grpc/external/org_golang_google_grpc/server.go:958 +0x136

Log preceding fatal error

*   | 	GOROOT/src/runtime/signal_unix.go:881
*   | github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotQueue).Admit
*   | 	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:165
*   | github.com/cockroachdb/cockroach/pkg/util/admission.(*SnapshotPacer).Pace
*   | 	github.com/cockroachdb/cockroach/pkg/util/admission/snapshot_queue.go:292
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*kvBatchSnapshotStrategy).Receive
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:918
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).receiveSnapshot
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_snapshot.go:1693
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot.func1
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:209
*   | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr
*   | 	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).HandleSnapshot
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/store_raft.go:206
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftSnapshot
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/pkg/kv/kvserver/raft_transport.go:638
*   | github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftSnapshot_Handler
*   | 	github.com/cockroachdb/cockroach/pkg/kv/kvserver/bazel-out/k8-fastbuild/bin/pkg/kv/kvserver/kvserver_go_proto_/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:267
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.StreamServerInterceptor.func13
*   | 	github.com/cockroachdb/cockroach/pkg/util/tracing/grpcinterceptor/grpc_interceptor.go:160
*   | google.golang.org/grpc.getChainStreamHandler.func1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func4
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:174
*   | google.golang.org/grpc.getChainStreamHandler.func1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481
*   | github.com/cockroachdb/cockroach/pkg/rpc.kvAuth.streamInterceptor
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/auth.go:152
*   | google.golang.org/grpc.getChainStreamHandler.func1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1481
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2.1
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:141
*   | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr
*   | 	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:349
*   | github.com/cockroachdb/cockroach/pkg/rpc.NewServerEx.func2
*   | 	github.com/cockroachdb/cockroach/pkg/rpc/pkg/rpc/context.go:140
*   | google.golang.org/grpc.NewServer.chainStreamServerInterceptors.chainStreamInterceptors.func2
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1472
*   | google.golang.org/grpc.(*Server).processStreamingRPC
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1636
*   | google.golang.org/grpc.(*Server).handleStream
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:1716
*   | google.golang.org/grpc.(*Server).serveStreams.func1.1
*   | 	google.golang.org/grpc/external/org_golang_google_grpc/server.go:947
*   | runtime.goexit
*   | 	src/runtime/asm_amd64.s:1695
* Wraps: (2) runtime error: invalid memory address or nil pointer dereference
* Error types: (1) *withstack.withStack (2) runtime.errorString
*

Parameters:

• attempt=1
• run=3
• shard=11

Help

See also: How To Investigate a Go Test Failure (internal)

Same failure on other branches

• sql: TestTenantGlobalAggregatedLivebytes failed #132962 sql: TestTenantGlobalAggregatedLivebytes failed [C-test-failure O-robot T-sql-queries branch-master release-blocker]

This test on roachdash | Improve this report!

Jira issue: CRDB-43388