From 64d341101ab6c6ba97bd17dd64953d8c464cb380 Mon Sep 17 00:00:00 2001 From: zhshy11 Date: Tue, 14 Sep 2021 11:11:26 +0800 Subject: [PATCH] Merge PR: auto dump pprof when cpu is high (#1018) * add auto dump pprof * add auto dump pprof * add auto dump pprof * add auto dump pprof * add auto dump pprof * add auto dump pprof: rm param * add auto dump pprof: add mem param * add auto dump pprof: GoroutineDump * add auto dump pprof: lrp * add auto dump pprof Co-authored-by: shaoyun.zhan@okcoin.com Co-authored-by: Zhong Qiu <36867992+zhongqiuwood@users.noreply.github.com> --- app/config/config.go | 3 ++ app/config/pprof.go | 81 ++++++++++++++++++++++++++++++++++++++++++ cmd/client/flags.go | 8 +++++ cmd/exchaind/replay.go | 12 +++++-- go.mod | 1 + go.sum | 3 ++ 6 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 app/config/pprof.go diff --git a/app/config/config.go b/app/config/config.go index 1dba633cb6..813635498f 100644 --- a/app/config/config.go +++ b/app/config/config.go @@ -64,6 +64,9 @@ func RegisterDynamicConfig() { // set the dynamic config oecConfig := GetOecConfig() tmconfig.SetDynamicConfig(oecConfig) + + //download pprof + PprofDownload() } func (c *OecConfig) loadFromConfig() { diff --git a/app/config/pprof.go b/app/config/pprof.go new file mode 100644 index 0000000000..43ff53c26c --- /dev/null +++ b/app/config/pprof.go @@ -0,0 +1,81 @@ +package config + +import ( + "github.com/mosn/holmes" + "github.com/spf13/viper" + "github.com/tendermint/tendermint/libs/cli" + tmos "github.com/tendermint/tendermint/libs/os" + "path" +) + +type PporfConfig struct { + autoDump bool + collectInterval string + coolDown string + dumpPath string + cpuTriggerPercentMin int + cpuTriggerPercentDiff int + cpuTriggerPercentAbs int + memTriggerPercentMin int + memTriggerPercentDiff int + memTriggerPercentAbs int +} + +const ( + FlagPprofAutoDump = "pprof-auto-dump" + FlagPprofCpuTriggerPercentMin = "pprof-cpu-trigger-percent-min" + FlagPprofCpuTriggerPercentDiff = "pprof-cpu-trigger-percent-diff" + FlagPprofCpuTriggerPercentAbs = "pprof-cpu-trigger-percent-abs" + FlagPprofMemTriggerPercentMin = "pprof-mem-trigger-percent-min" + FlagPprofMemTriggerPercentDiff = "pprof-mem-trigger-percent-diff" + FlagPprofMemTriggerPercentAbs = "pprof-mem-trigger-percent-abs" +) + +// PprofDownload auto dump pprof +func PprofDownload() { + c := LoadPprofFromConfig() + if !c.autoDump { + return + } + + h, err := holmes.New( + holmes.WithCollectInterval(c.collectInterval), + holmes.WithCoolDown(c.coolDown), + holmes.WithDumpPath(c.dumpPath), + holmes.WithCPUDump(c.cpuTriggerPercentMin, c.cpuTriggerPercentDiff, c.cpuTriggerPercentAbs), + holmes.WithMemDump(c.memTriggerPercentMin, c.memTriggerPercentDiff, c.memTriggerPercentAbs), + holmes.WithGoroutineDump(2000, 50, 5000), + holmes.WithBinaryDump(), + ) + if err != nil { + tmos.Exit(err.Error()) + } + h.EnableCPUDump().EnableMemDump().EnableGoroutineDump() + // start the metrics collect and dump loop + h.Start() +} + +func LoadPprofFromConfig() *PporfConfig { + autoDump := viper.GetBool(FlagPprofAutoDump) + dumpPath := path.Join(viper.GetString(cli.HomeFlag), "pprof") + cpuTriggerPercentMin := viper.GetInt(FlagPprofCpuTriggerPercentMin) + cpuTriggerPercentDiff := viper.GetInt(FlagPprofCpuTriggerPercentDiff) + cpuTriggerPercentAbs := viper.GetInt(FlagPprofCpuTriggerPercentAbs) + memTriggerPercentMin := viper.GetInt(FlagPprofMemTriggerPercentMin) + memTriggerPercentDiff := viper.GetInt(FlagPprofMemTriggerPercentDiff) + memTriggerPercentAbs := viper.GetInt(FlagPprofMemTriggerPercentAbs) + + c := &PporfConfig{ + autoDump: autoDump, + collectInterval: "5s", + coolDown: "3m", + dumpPath: dumpPath, + cpuTriggerPercentMin: cpuTriggerPercentMin, + cpuTriggerPercentDiff: cpuTriggerPercentDiff, + cpuTriggerPercentAbs: cpuTriggerPercentAbs, + memTriggerPercentMin: memTriggerPercentMin, + memTriggerPercentDiff: memTriggerPercentDiff, + memTriggerPercentAbs: memTriggerPercentAbs, + } + return c +} diff --git a/cmd/client/flags.go b/cmd/client/flags.go index 73f34a712d..933d665aa0 100644 --- a/cmd/client/flags.go +++ b/cmd/client/flags.go @@ -50,4 +50,12 @@ func RegisterAppFlag(cmd *cobra.Command) { cmd.Flags().Bool(config.FlagEnableDynamic, false, "Enable dynamic configuration for nodes") cmd.Flags().String(config.FlagApollo, "", "Apollo connection config(IP|AppID|NamespaceName) for dynamic configuration") + + cmd.Flags().Bool(config.FlagPprofAutoDump, false, "Enable auto dump pprof") + cmd.Flags().Int(config.FlagPprofCpuTriggerPercentMin, 45, "TriggerPercentMin of cpu to dump pprof") + cmd.Flags().Int(config.FlagPprofCpuTriggerPercentDiff, 50, "TriggerPercentDiff of cpu to dump pprof") + cmd.Flags().Int(config.FlagPprofCpuTriggerPercentAbs, 50, "TriggerPercentAbs of cpu to dump pprof") + cmd.Flags().Int(config.FlagPprofMemTriggerPercentMin, 70, "TriggerPercentMin of mem to dump pprof") + cmd.Flags().Int(config.FlagPprofMemTriggerPercentDiff, 50, "TriggerPercentDiff of mem to dump pprof") + cmd.Flags().Int(config.FlagPprofMemTriggerPercentAbs, 75, "TriggerPercentAbs of cpu mem dump pprof") } diff --git a/cmd/exchaind/replay.go b/cmd/exchaind/replay.go index 7815152a5c..78a10e3fae 100644 --- a/cmd/exchaind/replay.go +++ b/cmd/exchaind/replay.go @@ -3,6 +3,7 @@ package main import ( "fmt" storetypes "github.com/cosmos/cosmos-sdk/store/types" + "github.com/okex/exchain/app/config" "github.com/tendermint/tendermint/state" "log" "net/http" @@ -56,6 +57,13 @@ func replayCmd(ctx *server.Context) *cobra.Command { cmd.Flags().BoolVarP(&state.IgnoreSmbCheck, "ignore-smb", "i", false, "ignore state machine broken") cmd.Flags().String(server.FlagPruning, storetypes.PruningOptionNothing, "Pruning strategy (default|nothing|everything|custom)") cmd.Flags().Uint64(server.FlagHaltHeight, 0, "Block height at which to gracefully halt the chain and shutdown the node") + cmd.Flags().Bool(config.FlagPprofAutoDump, false, "Enable auto dump pprof") + cmd.Flags().Int(config.FlagPprofCpuTriggerPercentMin, 45, "TriggerPercentMin of cpu to dump pprof") + cmd.Flags().Int(config.FlagPprofCpuTriggerPercentDiff, 50, "TriggerPercentDiff of cpu to dump pprof") + cmd.Flags().Int(config.FlagPprofCpuTriggerPercentAbs, 50, "TriggerPercentAbs of cpu to dump pprof") + cmd.Flags().Int(config.FlagPprofMemTriggerPercentMin, 70, "TriggerPercentMin of mem to dump pprof") + cmd.Flags().Int(config.FlagPprofMemTriggerPercentDiff, 50, "TriggerPercentDiff of mem to dump pprof") + cmd.Flags().Int(config.FlagPprofMemTriggerPercentAbs, 75, "TriggerPercentAbs of cpu mem dump pprof") return cmd } @@ -91,7 +99,7 @@ func replayBlock(ctx *server.Context, originDataDir string) { startBlockHeight := currentBlockHeight + 1 //doReplay(ctx, state, stateStoreDB, proxyApp, originDataDir, startBlockHeight) haltBlockHeight := viper.GetInt64(server.FlagHaltHeight) - doReplay(ctx, state, stateStoreDB, proxyApp, originDataDir, startBlockHeight,haltBlockHeight) + doReplay(ctx, state, stateStoreDB, proxyApp, originDataDir, startBlockHeight, haltBlockHeight) } // panic if error is not nil @@ -165,7 +173,7 @@ func initChain(state sm.State, stateDB dbm.DB, genDoc *types.GenesisDoc, proxyAp } func doReplay(ctx *server.Context, state sm.State, stateStoreDB dbm.DB, - proxyApp proxy.AppConns, originDataDir string, startBlockHeight int64,haltBlockHeight int64){ + proxyApp proxy.AppConns, originDataDir string, startBlockHeight int64, haltBlockHeight int64) { originBlockStoreDB, err := openDB(blockStoreDB, originDataDir) panicError(err) originBlockStore := store.NewBlockStore(originBlockStoreDB) diff --git a/go.mod b/go.mod index 84932a20c6..ca1cf03482 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/json-iterator/go v1.1.9 github.com/kr/pretty v0.2.0 // indirect github.com/miguelmota/go-ethereum-hdwallet v0.0.0-20210614093730-56a4d342a6ff + github.com/mosn/holmes v0.0.0-20210830110104-685dc05437bf github.com/nacos-group/nacos-sdk-go v1.0.0 github.com/pierrec/lz4 v2.4.1+incompatible // indirect github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 40f5110dd6..c0bc0d9c1c 100644 --- a/go.sum +++ b/go.sum @@ -583,6 +583,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mosn/holmes v0.0.0-20210830110104-685dc05437bf h1:VfHb81w0uu11UCYRHZloSb5DHBoGyOwqs5LL981vemc= +github.com/mosn/holmes v0.0.0-20210830110104-685dc05437bf/go.mod h1:nzlLOLX+7+4VrlTn9kcZb+JBaOQtdtvCEQ4iqBAl5co= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= @@ -730,6 +732,7 @@ github.com/segmentio/kafka-go v0.2.2 h1:KIUln5unPisRL2yyAkZsDR/coiymN9Djunv6JKGQ github.com/segmentio/kafka-go v0.2.2/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v2.20.9+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=