From 920ff18bd8171dc0faa2e0bd154474e113ea35dc Mon Sep 17 00:00:00 2001 From: MengXiangJian <805442788@qq.com> Date: Sun, 11 Apr 2021 16:28:18 +0800 Subject: [PATCH] Merge PR: add pruning for block state db and add export fro app db (#811) * add cmd 'pruning' * add compact cmd * debug info * Update go.mod * upd pruneHeights * Update go.mod * Update go.mod * debug * Update pruning.go * Update go.mod * Update pruning.go * Update pruning.go * Update go.mod * Update pruning.go * cancel app state pruning * Update go.sum * cancel compact appdb * optimze * add export for application * add export for application * add export for application * clean code * add note for export app * change cosmos-sdk version * upd gomod Co-authored-by: ylsGit --- cmd/okexchaind/compact.go | 48 +++++++++++++++++++++ cmd/okexchaind/export_app.go | 46 ++++++++++++++++++++ cmd/okexchaind/main.go | 3 ++ cmd/okexchaind/pruning.go | 82 ++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 6 ++- 6 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 cmd/okexchaind/compact.go create mode 100644 cmd/okexchaind/export_app.go create mode 100644 cmd/okexchaind/pruning.go diff --git a/cmd/okexchaind/compact.go b/cmd/okexchaind/compact.go new file mode 100644 index 0000000000..86951d3e13 --- /dev/null +++ b/cmd/okexchaind/compact.go @@ -0,0 +1,48 @@ +package main + +import ( + "log" + "sync" + + "github.com/syndtr/goleveldb/leveldb/util" + dbm "github.com/tendermint/tm-db" + + "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/cosmos/cosmos-sdk/server" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/tendermint/tendermint/node" +) + +var wg sync.WaitGroup + +func compactCmd(ctx *server.Context) *cobra.Command { + cmd := &cobra.Command{ + Use: "compact", + Short: "Compact the leveldb", + RunE: func(cmd *cobra.Command, args []string) error { + config := ctx.Config + config.SetRoot(viper.GetString(flags.FlagHome)) + log.Println("--------- compact start ---------") + blockStoreDB, stateDB, _, err := initDBs(config, node.DefaultDBProvider) + if err != nil { + return err + } + + wg.Add(2) + go compactDB(blockStoreDB) + go compactDB(stateDB) + wg.Wait() + log.Println("--------- compact end ---------") + return nil + }, + } + + return cmd +} + +func compactDB(db dbm.DB) { + defer wg.Done() + err := db.(*dbm.GoLevelDB).DB().CompactRange(util.Range{}) + panicError(err) +} diff --git a/cmd/okexchaind/export_app.go b/cmd/okexchaind/export_app.go new file mode 100644 index 0000000000..fcd117c172 --- /dev/null +++ b/cmd/okexchaind/export_app.go @@ -0,0 +1,46 @@ +package main + +import ( + "log" + "path/filepath" + + "github.com/cosmos/cosmos-sdk/server" + "github.com/okex/okexchain/app" + "github.com/spf13/cobra" +) + +func exportAppCmd(ctx *server.Context) *cobra.Command { + cmd := &cobra.Command{ + Use: "export-app", + Short: "export current latest version of application db to new db into export dir", + Run: func(cmd *cobra.Command, args []string) { + log.Println("--------- export start ---------") + export(ctx) + log.Println("--------- export success ---------") + }, + } + return cmd +} + +// export current latest version of application db to new db into export dir +func export(ctx *server.Context) { + fromApp := createApp(ctx, "data") + toApp := createApp(ctx, "export") + + version := fromApp.LastCommitID().Version + log.Println("export app version ", version) + + err := fromApp.Export(toApp.BaseApp, version) + if err != nil { + panicError(err) + } +} + +func createApp(ctx *server.Context, dataPath string) *app.OKExChainApp { + rootDir := ctx.Config.RootDir + dataDir := filepath.Join(rootDir, dataPath) + db, err := openDB(applicationDB, dataDir) + panicError(err) + exapp := newApp(ctx.Logger, db, nil) + return exapp.(*app.OKExChainApp) +} diff --git a/cmd/okexchaind/main.go b/cmd/okexchaind/main.go index 63db342385..abae1ffaf5 100644 --- a/cmd/okexchaind/main.go +++ b/cmd/okexchaind/main.go @@ -81,6 +81,9 @@ func main() { // AddGenesisAccountCmd allows users to add accounts to the genesis file AddGenesisAccountCmd(ctx, cdc, app.DefaultNodeHome, app.DefaultCLIHome), flags.NewCompletionCmd(rootCmd, true), + pruningCmd(ctx), + compactCmd(ctx), + exportAppCmd(ctx), ) // Tendermint node base commands diff --git a/cmd/okexchaind/pruning.go b/cmd/okexchaind/pruning.go new file mode 100644 index 0000000000..0154f8f19d --- /dev/null +++ b/cmd/okexchaind/pruning.go @@ -0,0 +1,82 @@ +package main + +import ( + "fmt" + "log" + + "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/cosmos/cosmos-sdk/server" + "github.com/spf13/cobra" + "github.com/spf13/viper" + cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/node" + sm "github.com/tendermint/tendermint/state" + "github.com/tendermint/tendermint/store" + dbm "github.com/tendermint/tm-db" +) + +func pruningCmd(ctx *server.Context) *cobra.Command { + cmd := &cobra.Command{ + Use: "pruning", + Short: "Pruning blocks", + RunE: func(cmd *cobra.Command, args []string) error { + config := ctx.Config + config.SetRoot(viper.GetString(flags.FlagHome)) + log.Println("--------- pruning start ---------") + blockStoreDB, stateDB, _, err := initDBs(config, node.DefaultDBProvider) + if err != nil { + return err + } + + blockStore := store.NewBlockStore(blockStoreDB) + baseHeight := blockStore.Base() + size := blockStore.Size() + retainHeight := baseHeight + size - 2 + log.Printf("baseHeight:%d, size:%d, retainHeight:%d\n", baseHeight, size, retainHeight) + + pruneBlocks(blockStore, stateDB, retainHeight) + + log.Println("--------- pruning end ---------") + return nil + }, + } + + return cmd +} + +func initDBs(config *cfg.Config, dbProvider node.DBProvider) (blockStoreDB, stateDB, appDB dbm.DB, err error) { + blockStoreDB, err = dbProvider(&node.DBContext{"blockstore", config}) + if err != nil { + return + } + + stateDB, err = dbProvider(&node.DBContext{"state", config}) + if err != nil { + return + } + + appDB, err = dbProvider(&node.DBContext{"application", config}) + if err != nil { + return + } + + return +} + +func pruneBlocks(blockStore *store.BlockStore, stateDB dbm.DB, retainHeight int64) { + base := blockStore.Base() + if retainHeight <= base { + return + } + pruned, err := blockStore.PruneBlocks(retainHeight) + if err != nil { + panic(fmt.Errorf("failed to prune block store: %w", err)) + } + err = sm.PruneStates(stateDB, base, retainHeight) + if err != nil { + panic(fmt.Errorf("failed to prune state database: %w", err)) + } + + log.Printf("pruned blocks: %d, retainHeight: %d\n", pruned, retainHeight) + log.Printf("block store base: %d, block store size: %d\n", blockStore.Base(), blockStore.Size()) +} diff --git a/go.mod b/go.mod index 2cc4a7038a..5efbe0d116 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/okex/cosmos-sdk v0.39.2-exchain + github.com/cosmos/cosmos-sdk => github.com/okex/cosmos-sdk v0.39.3-0.20210411062155-2683537970bb github.com/tendermint/iavl => github.com/okex/iavl v0.14.3-exchain github.com/tendermint/tendermint => github.com/okex/tendermint v0.33.9-exchain ) diff --git a/go.sum b/go.sum index 48575e4fcd..d0b3babd79 100644 --- a/go.sum +++ b/go.sum @@ -500,9 +500,11 @@ github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/okex/cosmos-sdk v0.39.2-exchain h1:PN8xdkn0WTX8sqmvjCdNFhT2zRZ8Q+YwM1sGMi0gX8s= -github.com/okex/cosmos-sdk v0.39.2-exchain/go.mod h1:Y1C2roqCVZ6gSYG31X77x4NLcbSGS9VP1GIiAE1imcc= +github.com/okex/cosmos-sdk v0.39.3-0.20210411062155-2683537970bb h1:lpqIwiS+F0RoIpiviIp0WWvAdYiEziSw39A5XIBJGOg= +github.com/okex/cosmos-sdk v0.39.3-0.20210411062155-2683537970bb/go.mod h1:Y1C2roqCVZ6gSYG31X77x4NLcbSGS9VP1GIiAE1imcc= github.com/okex/iavl v0.14.3-exchain h1:kwRIwpFD6B8mDDqoaxeUN3Pg2GW0Vr+sA+b86renWcA= +github.com/okex/iavl v0.14.3-exchain h1:kwRIwpFD6B8mDDqoaxeUN3Pg2GW0Vr+sA+b86renWcA= +github.com/okex/iavl v0.14.3-exchain/go.mod h1:vHLYxU/zuxBmxxr1v+5Vnd/JzcIsyK17n9P9RDubPVU= github.com/okex/iavl v0.14.3-exchain/go.mod h1:vHLYxU/zuxBmxxr1v+5Vnd/JzcIsyK17n9P9RDubPVU= github.com/okex/tendermint v0.33.9-exchain h1:8JuPeB+NgfnPFa5ki89bspLAJzWeIV9C6OFSr1u5IsI= github.com/okex/tendermint v0.33.9-exchain/go.mod h1:EoGTbJUufUueNIigY3zyO6f7GOj29OdpFhuR8sxWdSU=