From 5a4751f25d55c931eb67d31bccc3268560d166db Mon Sep 17 00:00:00 2001 From: Cyril Galibern Date: Mon, 21 Aug 2023 10:02:00 +0200 Subject: [PATCH] [cmd] Fix 'om mon -w' race ## crash detected with: go build without --race ### goroutine 77 [running]: github.com/opensvc/om3/core/cluster.Frame.sObject({{0x0, 0x0, 0x0}, {0x34da020, 0x0, 0x0}, {{{{...}, {...}, {...}, {...}, ...}, ...}, ...}, ...}, ...) /root/omg/core/cluster/frame_objects.go:105 +0x55 github.com/opensvc/om3/core/cluster.Frame.wObjects({{0x0, 0x0, 0x0}, {0x34da020, 0x0, 0x0}, {{{{...}, {...}, {...}, {...}, ...}, ...}, ...}, ...}) /root/omg/core/cluster/frame_objects.go:18 +0x24a github.com/opensvc/om3/core/cluster.(*Frame).Render(0xc005341e80) /root/omg/core/cluster/frame.go:110 +0x470 github.com/opensvc/om3/core/monitor.(*T).doOneShot.func1() /root/omg/core/monitor/main.go:137 +0xc5 github.com/opensvc/om3/core/output.Renderer.Sprint({{0x15c3b61, 0x4}, {0x15c3b61, 0x4}, {0x148ea80, 0xc004f51000}, 0xc0052ab008, 0xc00007e1c0, 0x0}) /root/omg/core/output/renderer.go:121 +0x79a github.com/opensvc/om3/core/monitor.(*T).doOneShot(_, {{{{0xc004d681e0, 0x24}, {0xc004d71de0, 0x4}, {0xc004d774c0, 0x3, 0x4}, {0xc004d77500, 0x3, ...}, ...}, ...}, ...}, ...) /root/omg/core/monitor/main.go:146 +0x1e5 github.com/opensvc/om3/core/monitor.(*T).watch.func2(0xc004c83c00?) /root/omg/core/monitor/main.go:211 +0x105 created by github.com/opensvc/om3/core/monitor.(*T).watch /root/omg/core/monitor/main.go:209 +0x3d1 ### goroutine 1 [select]: github.com/opensvc/om3/core/monitor.(*T).watch(0xc003eccd80, {0x2a0fe60, 0xc00530c1e0}, {0x2a147a0?, 0xc004df92c0}, {0x2a114e0?, 0xc00011e008}) /root/omg/core/monitor/main.go:225 +0x5b0 github.com/opensvc/om3/core/monitor.(*T).DoWatch(...) /root/omg/core/monitor/main.go:160 github.com/opensvc/om3/core/commands.(*CmdObjectMonitor).Run(0xc000236ea0, {0x2a07708?, 0x0?}, {0x0?, 0x0?}) /root/omg/core/commands/object_monitor.go:42 +0x4ff github.com/opensvc/om3/cmd.newCmdMonitor.func1(0xc00026d180?, {0x15c3e21?, 0x1?, 0x1?}) /root/omg/cmd/factory.go:465 +0x2d github.com/spf13/cobra.(*Command).execute(0xc00026d180, {0xc0001112f0, 0x1, 0x1}) /root/go/pkg/mod/github.com/spf13/cobra@v1.5.0/command.go:872 +0x694 github.com/spf13/cobra.(*Command).ExecuteC(0x3467560) /root/go/pkg/mod/github.com/spf13/cobra@v1.5.0/command.go:990 +0x3bd github.com/spf13/cobra.(*Command).Execute(...) /root/go/pkg/mod/github.com/spf13/cobra@v1.5.0/command.go:918 github.com/opensvc/om3/cmd.ExecuteArgs({0xc000112160?, 0x1371f20?, 0xc000006340?}) /root/omg/cmd/root.go:189 +0x2a github.com/opensvc/om3/cmd.Execute(...) /root/omg/cmd/root.go:139 main.main() /root/omg/main.go:8 +0x4e ### goroutine 19 [select]: github.com/opensvc/om3/core/rawconfig.init.0.func1() /root/omg/core/rawconfig/node.go:96 +0xcc created by github.com/opensvc/om3/core/rawconfig.init.0 /root/omg/core/rawconfig/node.go:93 +0x6a ### goroutine 5 [select]: github.com/opensvc/om3/util/httpclientcache.server() /root/omg/util/httpclientcache/main.go:79 +0x10f created by github.com/opensvc/om3/util/httpclientcache.init.0 /root/omg/util/httpclientcache/main.go:73 +0x25 ### goroutine 85 [select]: github.com/opensvc/om3/core/event/sseevent.(*ReadCloser).Read(0xc004df92c0) /root/omg/core/event/sseevent/main.go:104 +0x116 github.com/opensvc/om3/core/monitor.(*T).watch.func1() /root/omg/core/monitor/main.go:189 +0x122 created by github.com/opensvc/om3/core/monitor.(*T).watch /root/omg/core/monitor/main.go:183 +0x1f8 ### goroutine 87 [sync.Cond.Wait]: sync.runtime_notifyListWait(0xc0049ebb48, 0x2f) /usr/local/go/src/runtime/sema.go:527 +0x14c sync.(*Cond).Wait(0xc0052b6340?) /usr/local/go/src/sync/cond.go:70 +0x8c golang.org/x/net/http2.(*pipe).Read(0xc0049ebb30, {0xc00530b6b2, 0x94e, 0x94e}) /root/go/pkg/mod/golang.org/x/net@v0.7.0/http2/pipe.go:76 +0xeb golang.org/x/net/http2.transportResponseBody.Read({0xc00530b6b1?}, {0xc00530b6b2?, 0x94f?, 0x0?}) /root/go/pkg/mod/golang.org/x/net@v0.7.0/http2/transport.go:2507 +0x75 bufio.(*Scanner).Scan(0xc000075f20) /usr/local/go/src/bufio/scan.go:214 +0x876 github.com/opensvc/om3/core/event/sseevent.(*ReadCloser).parse(0xc004df92c0) /root/omg/core/event/sseevent/main.go:146 +0x1c5 created by github.com/opensvc/om3/core/event/sseevent.(*ReadCloser).Read /root/omg/core/event/sseevent/main.go:101 +0x85 ### goroutine 76 [select]: golang.org/x/net/http2.(*clientStream).writeRequest(0xc0049ebb00, 0xc004d62400) /root/go/pkg/mod/golang.org/x/net@v0.7.0/http2/transport.go:1437 +0xb27 golang.org/x/net/http2.(*clientStream).doRequest(0x10320aa?, 0xc0045efc00?) /root/go/pkg/mod/golang.org/x/net@v0.7.0/http2/transport.go:1299 +0x1e created by golang.org/x/net/http2.(*ClientConn).RoundTrip /root/go/pkg/mod/golang.org/x/net@v0.7.0/http2/transport.go:1228 +0x34a ### goroutine 75 [IO wait]: internal/poll.runtime_pollWait(0x7f4624c50158, 0x72) /usr/local/go/src/runtime/netpoll.go:306 +0x89 internal/poll.(*pollDesc).wait(0xc004d09480?, 0xc005092000?, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc004d09480, {0xc005092000, 0x1000, 0x1000}) /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 net.(*netFD).Read(0xc004d09480, {0xc005092000?, 0x44eb94?, 0x30000008b?}) /usr/local/go/src/net/fd_posix.go:55 +0x29 net.(*conn).Read(0xc00011e450, {0xc005092000?, 0xc0049ebb50?, 0xc000074c98?}) /usr/local/go/src/net/net.go:183 +0x45 bufio.(*Reader).Read(0xc004cf8540, {0xc004d544a0, 0x9, 0xc000074d38?}) /usr/local/go/src/bufio/bufio.go:237 +0x1bb io.ReadAtLeast({0x2a0ee00, 0xc004cf8540}, {0xc004d544a0, 0x9, 0x9}, 0x9) /usr/local/go/src/io/io.go:332 +0x9a io.ReadFull(...) /usr/local/go/src/io/io.go:351 golang.org/x/net/http2.readFrameHeader({0xc004d544a0?, 0x9?, 0xc000000000?}, {0x2a0ee00?, 0xc004cf8540?}) /root/go/pkg/mod/golang.org/x/net@v0.7.0/http2/frame.go:237 +0x6e golang.org/x/net/http2.(*Framer).ReadFrame(0xc004d54460) /root/go/pkg/mod/golang.org/x/net@v0.7.0/http2/frame.go:498 +0x95 golang.org/x/net/http2.(*clientConnReadLoop).run(0xc000074f98) /root/go/pkg/mod/golang.org/x/net@v0.7.0/http2/transport.go:2224 +0x12e golang.org/x/net/http2.(*ClientConn).readLoop(0xc0049eb980) /root/go/pkg/mod/golang.org/x/net@v0.7.0/http2/transport.go:2119 +0x6f created by golang.org/x/net/http2.(*Transport).newClientConn /root/go/pkg/mod/golang.org/x/net@v0.7.0/http2/transport.go:817 +0xc1f ## crash with go-race WARNING: DATA RACE # fatal error: concurrent map read and map write ### Write at 0x00c00006d1d8 by main goroutine: github.com/opensvc/om3/daemon/msgbus.(*ClusterData).onDaemonHb() daemon/msgbus/daemon_hb.go:4 +0x264 github.com/opensvc/om3/daemon/msgbus.(*ClusterData).ApplyMessage() daemon/msgbus/main.go:24 +0x200 github.com/opensvc/om3/core/monitor.(*T).watch() core/monitor/main.go:239 +0xbe6 github.com/opensvc/om3/core/monitor.(*T).DoWatch() core/monitor/main.go:160 +0x829 github.com/opensvc/om3/core/commands.(*CmdObjectMonitor).Run() core/commands/object_monitor.go:42 +0x725 github.com/opensvc/om3/cmd.newCmdMonitor.func1() cmd/factory.go:465 +0x44 github.com/spf13/cobra.(*Command).execute() /root/go/pkg/mod/github.com/spf13/cobra@v1.5.0/command.go:872 +0xaf5 github.com/spf13/cobra.(*Command).ExecuteC() /root/go/pkg/mod/github.com/spf13/cobra@v1.5.0/command.go:990 +0x5da github.com/spf13/cobra.(*Command).Execute() /root/go/pkg/mod/github.com/spf13/cobra@v1.5.0/command.go:918 +0x5c github.com/opensvc/om3/cmd.ExecuteArgs() cmd/root.go:189 +0x45 github.com/opensvc/om3/cmd.Execute() cmd/root.go:139 +0x6b main.main() main.go:8 +0x25 runtime.main() /usr/local/go/src/runtime/proc.go:250 +0x206 github.com/cilium/ebpf/link.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/link/syscalls.go:50 +0x224 github.com/cilium/ebpf/link.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/link/syscalls.go:29 +0x1c4 runtime.doInit() /usr/local/go/src/runtime/proc.go:6506 +0x125 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:438 +0x3b0 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:412 +0x350 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:390 +0x2f0 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:372 +0x290 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:191 +0x230 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:175 +0x1d0 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:158 +0x170 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:140 +0x110 runtime.doInit() /usr/local/go/src/runtime/proc.go:6506 +0x125 github.com/cilium/ebpf/internal/btf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/internal/btf/btf.go:727 +0xaa ### Previous read at 0x00c00006d1d8 by goroutine 36: github.com/opensvc/om3/core/monitor.(*T).watch.func2() core/monitor/main.go:211 +0xdb github.com/opensvc/om3/core/monitor.(*T).watch.func4() core/monitor/main.go:216 +0x47 ### Goroutine 36 (running) created at: github.com/opensvc/om3/core/monitor.(*T).watch() core/monitor/main.go:209 +0x651 github.com/opensvc/om3/core/monitor.(*T).DoWatch() core/monitor/main.go:160 +0x829 github.com/opensvc/om3/core/commands.(*CmdObjectMonitor).Run() core/commands/object_monitor.go:42 +0x725 github.com/opensvc/om3/cmd.newCmdMonitor.func1() cmd/factory.go:465 +0x44 github.com/spf13/cobra.(*Command).execute() /root/go/pkg/mod/github.com/spf13/cobra@v1.5.0/command.go:872 +0xaf5 github.com/spf13/cobra.(*Command).ExecuteC() /root/go/pkg/mod/github.com/spf13/cobra@v1.5.0/command.go:990 +0x5da github.com/spf13/cobra.(*Command).Execute() /root/go/pkg/mod/github.com/spf13/cobra@v1.5.0/command.go:918 +0x5c github.com/opensvc/om3/cmd.ExecuteArgs() cmd/root.go:189 +0x45 github.com/opensvc/om3/cmd.Execute() cmd/root.go:139 +0x6b main.main() main.go:8 +0x25 runtime.main() /usr/local/go/src/runtime/proc.go:250 +0x206 github.com/cilium/ebpf/link.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/link/syscalls.go:50 +0x224 github.com/cilium/ebpf/link.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/link/syscalls.go:29 +0x1c4 runtime.doInit() /usr/local/go/src/runtime/proc.go:6506 +0x125 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:438 +0x3b0 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:412 +0x350 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:390 +0x2f0 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:372 +0x290 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:191 +0x230 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:175 +0x1d0 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:158 +0x170 github.com/cilium/ebpf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/syscalls.go:140 +0x110 runtime.doInit() /usr/local/go/src/runtime/proc.go:6506 +0x125 github.com/cilium/ebpf/internal/btf.init() /root/go/pkg/mod/github.com/cilium/ebpf@v0.7.0/internal/btf/btf.go:727 +0xaa --- core/monitor/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/monitor/main.go b/core/monitor/main.go index aa8379ede..9a566e3d5 100644 --- a/core/monitor/main.go +++ b/core/monitor/main.go @@ -213,7 +213,7 @@ func (m *T) watch(statusGetter Getter, evReader event.ReadCloser, out io.Writer) for d := range dataC { m.doOneShot(*d, true, out) } - }(data) + }(data.DeepCopy()) defer close(dataC)