Skip to content

Commit

Permalink
[cmd] Fix 'om mon -w' race
Browse files Browse the repository at this point in the history 
## crash detected with: go build without --race
### goroutine 77 [running]:
    github.com/opensvc/om3/core/cluster.Frame.sObject({{0x0, 0x0, 0x0}, {0x34da020, 0x0, 0x0}, {{{{...}, {...}, {...}, {...}, ...}, ...}, ...}, ...}, ...)
        /root/omg/core/cluster/frame_objects.go:105 +0x55
    github.com/opensvc/om3/core/cluster.Frame.wObjects({{0x0, 0x0, 0x0}, {0x34da020, 0x0, 0x0}, {{{{...}, {...}, {...}, {...}, ...}, ...}, ...}, ...})
        /root/omg/core/cluster/frame_objects.go:18 +0x24a
    github.com/opensvc/om3/core/cluster.(*Frame).Render(0xc005341e80)
        /root/omg/core/cluster/frame.go:110 +0x470
    github.com/opensvc/om3/core/monitor.(*T).doOneShot.func1()
        /root/omg/core/monitor/main.go:137 +0xc5
    github.com/opensvc/om3/core/output.Renderer.Sprint({{0x15c3b61, 0x4}, {0x15c3b61, 0x4}, {0x148ea80, 0xc004f51000}, 0xc0052ab008, 0xc00007e1c0, 0x0})
        /root/omg/core/output/renderer.go:121 +0x79a
    github.com/opensvc/om3/core/monitor.(*T).doOneShot(_, {{{{0xc004d681e0, 0x24}, {0xc004d71de0, 0x4}, {0xc004d774c0, 0x3, 0x4}, {0xc004d77500, 0x3, ...}, ...}, ...}, ...}, ...)
        /root/omg/core/monitor/main.go:146 +0x1e5
    github.com/opensvc/om3/core/monitor.(*T).watch.func2(0xc004c83c00?)
        /root/omg/core/monitor/main.go:211 +0x105
    created by github.com/opensvc/om3/core/monitor.(*T).watch
        /root/omg/core/monitor/main.go:209 +0x3d1

### goroutine 1 [select]:
    github.com/opensvc/om3/core/monitor.(*T).watch(0xc003eccd80, {0x2a0fe60, 0xc00530c1e0}, {0x2a147a0?, 0xc004df92c0}, {0x2a114e0?, 0xc00011e008})
        /root/omg/core/monitor/main.go:225 +0x5b0
    github.com/opensvc/om3/core/monitor.(*T).DoWatch(...)
        /root/omg/core/monitor/main.go:160
    github.com/opensvc/om3/core/commands.(*CmdObjectMonitor).Run(0xc000236ea0, {0x2a07708?, 0x0?}, {0x0?, 0x0?})
        /root/omg/core/commands/object_monitor.go:42 +0x4ff
    github.com/opensvc/om3/cmd.newCmdMonitor.func1(0xc00026d180?, {0x15c3e21?, 0x1?, 0x1?})
        /root/omg/cmd/factory.go:465 +0x2d
    github.com/spf13/cobra.(*Command).execute(0xc00026d180, {0xc0001112f0, 0x1, 0x1})
        /root/go/pkg/mod/github.com/spf13/[email protected]/command.go:872 +0x694
    github.com/spf13/cobra.(*Command).ExecuteC(0x3467560)
        /root/go/pkg/mod/github.com/spf13/[email protected]/command.go:990 +0x3bd
    github.com/spf13/cobra.(*Command).Execute(...)
        /root/go/pkg/mod/github.com/spf13/[email protected]/command.go:918
    github.com/opensvc/om3/cmd.ExecuteArgs({0xc000112160?, 0x1371f20?, 0xc000006340?})
        /root/omg/cmd/root.go:189 +0x2a
    github.com/opensvc/om3/cmd.Execute(...)
        /root/omg/cmd/root.go:139
    main.main()
        /root/omg/main.go:8 +0x4e

### goroutine 19 [select]:
    github.com/opensvc/om3/core/rawconfig.init.0.func1()
        /root/omg/core/rawconfig/node.go:96 +0xcc
    created by github.com/opensvc/om3/core/rawconfig.init.0
        /root/omg/core/rawconfig/node.go:93 +0x6a

### goroutine 5 [select]:
    github.com/opensvc/om3/util/httpclientcache.server()
        /root/omg/util/httpclientcache/main.go:79 +0x10f
    created by github.com/opensvc/om3/util/httpclientcache.init.0
        /root/omg/util/httpclientcache/main.go:73 +0x25

### goroutine 85 [select]:
    github.com/opensvc/om3/core/event/sseevent.(*ReadCloser).Read(0xc004df92c0)
        /root/omg/core/event/sseevent/main.go:104 +0x116
    github.com/opensvc/om3/core/monitor.(*T).watch.func1()
        /root/omg/core/monitor/main.go:189 +0x122
    created by github.com/opensvc/om3/core/monitor.(*T).watch
        /root/omg/core/monitor/main.go:183 +0x1f8

### goroutine 87 [sync.Cond.Wait]:
    sync.runtime_notifyListWait(0xc0049ebb48, 0x2f)
        /usr/local/go/src/runtime/sema.go:527 +0x14c
    sync.(*Cond).Wait(0xc0052b6340?)
        /usr/local/go/src/sync/cond.go:70 +0x8c
    golang.org/x/net/http2.(*pipe).Read(0xc0049ebb30, {0xc00530b6b2, 0x94e, 0x94e})
        /root/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:76 +0xeb
    golang.org/x/net/http2.transportResponseBody.Read({0xc00530b6b1?}, {0xc00530b6b2?, 0x94f?, 0x0?})
        /root/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2507 +0x75
    bufio.(*Scanner).Scan(0xc000075f20)
        /usr/local/go/src/bufio/scan.go:214 +0x876
    github.com/opensvc/om3/core/event/sseevent.(*ReadCloser).parse(0xc004df92c0)
        /root/omg/core/event/sseevent/main.go:146 +0x1c5
    created by github.com/opensvc/om3/core/event/sseevent.(*ReadCloser).Read
        /root/omg/core/event/sseevent/main.go:101 +0x85

### goroutine 76 [select]:
    golang.org/x/net/http2.(*clientStream).writeRequest(0xc0049ebb00, 0xc004d62400)
        /root/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1437 +0xb27
    golang.org/x/net/http2.(*clientStream).doRequest(0x10320aa?, 0xc0045efc00?)
        /root/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1299 +0x1e
    created by golang.org/x/net/http2.(*ClientConn).RoundTrip
        /root/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1228 +0x34a

### goroutine 75 [IO wait]:
    internal/poll.runtime_pollWait(0x7f4624c50158, 0x72)
        /usr/local/go/src/runtime/netpoll.go:306 +0x89
    internal/poll.(*pollDesc).wait(0xc004d09480?, 0xc005092000?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32
    internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
    internal/poll.(*FD).Read(0xc004d09480, {0xc005092000, 0x1000, 0x1000})
        /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299
    net.(*netFD).Read(0xc004d09480, {0xc005092000?, 0x44eb94?, 0x30000008b?})
        /usr/local/go/src/net/fd_posix.go:55 +0x29
    net.(*conn).Read(0xc00011e450, {0xc005092000?, 0xc0049ebb50?, 0xc000074c98?})
        /usr/local/go/src/net/net.go:183 +0x45
    bufio.(*Reader).Read(0xc004cf8540, {0xc004d544a0, 0x9, 0xc000074d38?})
        /usr/local/go/src/bufio/bufio.go:237 +0x1bb
    io.ReadAtLeast({0x2a0ee00, 0xc004cf8540}, {0xc004d544a0, 0x9, 0x9}, 0x9)
        /usr/local/go/src/io/io.go:332 +0x9a
    io.ReadFull(...)
        /usr/local/go/src/io/io.go:351
    golang.org/x/net/http2.readFrameHeader({0xc004d544a0?, 0x9?, 0xc000000000?}, {0x2a0ee00?, 0xc004cf8540?})
        /root/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x6e
    golang.org/x/net/http2.(*Framer).ReadFrame(0xc004d54460)
        /root/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:498 +0x95
    golang.org/x/net/http2.(*clientConnReadLoop).run(0xc000074f98)
        /root/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2224 +0x12e
    golang.org/x/net/http2.(*ClientConn).readLoop(0xc0049eb980)
        /root/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2119 +0x6f
    created by golang.org/x/net/http2.(*Transport).newClientConn
        /root/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:817 +0xc1f

## crash with go-race

    WARNING: DATA RACE
    # fatal error: concurrent map read and map write

### Write at 0x00c00006d1d8 by main goroutine:
  github.com/opensvc/om3/daemon/msgbus.(*ClusterData).onDaemonHb()
      daemon/msgbus/daemon_hb.go:4 +0x264
  github.com/opensvc/om3/daemon/msgbus.(*ClusterData).ApplyMessage()
      daemon/msgbus/main.go:24 +0x200
  github.com/opensvc/om3/core/monitor.(*T).watch()
      core/monitor/main.go:239 +0xbe6
  github.com/opensvc/om3/core/monitor.(*T).DoWatch()
      core/monitor/main.go:160 +0x829
  github.com/opensvc/om3/core/commands.(*CmdObjectMonitor).Run()
      core/commands/object_monitor.go:42 +0x725
  github.com/opensvc/om3/cmd.newCmdMonitor.func1()
      cmd/factory.go:465 +0x44
  github.com/spf13/cobra.(*Command).execute()
      /root/go/pkg/mod/github.com/spf13/[email protected]/command.go:872 +0xaf5
  github.com/spf13/cobra.(*Command).ExecuteC()
      /root/go/pkg/mod/github.com/spf13/[email protected]/command.go:990 +0x5da
  github.com/spf13/cobra.(*Command).Execute()
      /root/go/pkg/mod/github.com/spf13/[email protected]/command.go:918 +0x5c
  github.com/opensvc/om3/cmd.ExecuteArgs()
      cmd/root.go:189 +0x45
  github.com/opensvc/om3/cmd.Execute()
      cmd/root.go:139 +0x6b
  main.main()
      main.go:8 +0x25
  runtime.main()
      /usr/local/go/src/runtime/proc.go:250 +0x206
  github.com/cilium/ebpf/link.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/link/syscalls.go:50 +0x224
  github.com/cilium/ebpf/link.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/link/syscalls.go:29 +0x1c4
  runtime.doInit()
      /usr/local/go/src/runtime/proc.go:6506 +0x125
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:438 +0x3b0
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:412 +0x350
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:390 +0x2f0
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:372 +0x290
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:191 +0x230
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:175 +0x1d0
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:158 +0x170
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:140 +0x110
  runtime.doInit()
      /usr/local/go/src/runtime/proc.go:6506 +0x125
  github.com/cilium/ebpf/internal/btf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/internal/btf/btf.go:727 +0xaa

### Previous read at 0x00c00006d1d8 by goroutine 36:
  github.com/opensvc/om3/core/monitor.(*T).watch.func2()
      core/monitor/main.go:211 +0xdb
  github.com/opensvc/om3/core/monitor.(*T).watch.func4()
      core/monitor/main.go:216 +0x47

### Goroutine 36 (running) created at:
  github.com/opensvc/om3/core/monitor.(*T).watch()
      core/monitor/main.go:209 +0x651
  github.com/opensvc/om3/core/monitor.(*T).DoWatch()
      core/monitor/main.go:160 +0x829
  github.com/opensvc/om3/core/commands.(*CmdObjectMonitor).Run()
      core/commands/object_monitor.go:42 +0x725
  github.com/opensvc/om3/cmd.newCmdMonitor.func1()
      cmd/factory.go:465 +0x44
  github.com/spf13/cobra.(*Command).execute()
      /root/go/pkg/mod/github.com/spf13/[email protected]/command.go:872 +0xaf5
  github.com/spf13/cobra.(*Command).ExecuteC()
      /root/go/pkg/mod/github.com/spf13/[email protected]/command.go:990 +0x5da
  github.com/spf13/cobra.(*Command).Execute()
      /root/go/pkg/mod/github.com/spf13/[email protected]/command.go:918 +0x5c
  github.com/opensvc/om3/cmd.ExecuteArgs()
      cmd/root.go:189 +0x45
  github.com/opensvc/om3/cmd.Execute()
      cmd/root.go:139 +0x6b
  main.main()
      main.go:8 +0x25
  runtime.main()
      /usr/local/go/src/runtime/proc.go:250 +0x206
  github.com/cilium/ebpf/link.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/link/syscalls.go:50 +0x224
  github.com/cilium/ebpf/link.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/link/syscalls.go:29 +0x1c4
  runtime.doInit()
      /usr/local/go/src/runtime/proc.go:6506 +0x125
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:438 +0x3b0
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:412 +0x350
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:390 +0x2f0
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:372 +0x290
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:191 +0x230
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:175 +0x1d0
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:158 +0x170
  github.com/cilium/ebpf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/syscalls.go:140 +0x110
  runtime.doInit()
      /usr/local/go/src/runtime/proc.go:6506 +0x125
  github.com/cilium/ebpf/internal/btf.init()
      /root/go/pkg/mod/github.com/cilium/[email protected]/internal/btf/btf.go:727 +0xaa
  • Loading branch information
@cgalibern
cgalibern committed Aug 21, 2023
1 parent 1c31491 commit 5a4751f
Showing 1 changed file with 1 addition and 1 deletion.
2 changes: 1 addition & 1 deletion core/monitor/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ func (m *T) watch(statusGetter Getter, evReader event.ReadCloser, out io.Writer)
for d := range dataC {
m.doOneShot(*d, true, out)
}
}(data)
}(data.DeepCopy())

defer close(dataC)

Expand Down

0 comments on commit 5a4751f

Please sign in to comment.