-
Notifications
You must be signed in to change notification settings - Fork 185
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge PR: add pruning for block state db and add export fro app db (#811
) * add cmd 'pruning' * add compact cmd * debug info * Update go.mod * upd pruneHeights * Update go.mod * Update go.mod * debug * Update pruning.go * Update go.mod * Update pruning.go * Update pruning.go * Update go.mod * Update pruning.go * cancel app state pruning * Update go.sum * cancel compact appdb * optimze * add export for application * add export for application * add export for application * clean code * add note for export app * change cosmos-sdk version * upd gomod Co-authored-by: ylsGit <[email protected]>
- Loading branch information
1 parent
13f635e
commit 920ff18
Showing
6 changed files
with
184 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "log" | ||
| "sync" | ||
|
|
||
| "github.com/syndtr/goleveldb/leveldb/util" | ||
| dbm "github.com/tendermint/tm-db" | ||
|
|
||
| "github.com/cosmos/cosmos-sdk/client/flags" | ||
| "github.com/cosmos/cosmos-sdk/server" | ||
| "github.com/spf13/cobra" | ||
| "github.com/spf13/viper" | ||
| "github.com/tendermint/tendermint/node" | ||
| ) | ||
|
|
||
| var wg sync.WaitGroup | ||
|
|
||
| func compactCmd(ctx *server.Context) *cobra.Command { | ||
| cmd := &cobra.Command{ | ||
| Use: "compact", | ||
| Short: "Compact the leveldb", | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| config := ctx.Config | ||
| config.SetRoot(viper.GetString(flags.FlagHome)) | ||
| log.Println("--------- compact start ---------") | ||
| blockStoreDB, stateDB, _, err := initDBs(config, node.DefaultDBProvider) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| wg.Add(2) | ||
| go compactDB(blockStoreDB) | ||
| go compactDB(stateDB) | ||
| wg.Wait() | ||
| log.Println("--------- compact end ---------") | ||
| return nil | ||
| }, | ||
| } | ||
|
|
||
| return cmd | ||
| } | ||
|
|
||
| func compactDB(db dbm.DB) { | ||
| defer wg.Done() | ||
| err := db.(*dbm.GoLevelDB).DB().CompactRange(util.Range{}) | ||
| panicError(err) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "log" | ||
| "path/filepath" | ||
|
|
||
| "github.com/cosmos/cosmos-sdk/server" | ||
| "github.com/okex/okexchain/app" | ||
| "github.com/spf13/cobra" | ||
| ) | ||
|
|
||
| func exportAppCmd(ctx *server.Context) *cobra.Command { | ||
| cmd := &cobra.Command{ | ||
| Use: "export-app", | ||
| Short: "export current latest version of application db to new db into export dir", | ||
| Run: func(cmd *cobra.Command, args []string) { | ||
| log.Println("--------- export start ---------") | ||
| export(ctx) | ||
| log.Println("--------- export success ---------") | ||
| }, | ||
| } | ||
| return cmd | ||
| } | ||
|
|
||
| // export current latest version of application db to new db into export dir | ||
| func export(ctx *server.Context) { | ||
| fromApp := createApp(ctx, "data") | ||
| toApp := createApp(ctx, "export") | ||
|
|
||
| version := fromApp.LastCommitID().Version | ||
| log.Println("export app version ", version) | ||
|
|
||
| err := fromApp.Export(toApp.BaseApp, version) | ||
| if err != nil { | ||
| panicError(err) | ||
| } | ||
| } | ||
|
|
||
| func createApp(ctx *server.Context, dataPath string) *app.OKExChainApp { | ||
| rootDir := ctx.Config.RootDir | ||
| dataDir := filepath.Join(rootDir, dataPath) | ||
| db, err := openDB(applicationDB, dataDir) | ||
| panicError(err) | ||
| exapp := newApp(ctx.Logger, db, nil) | ||
| return exapp.(*app.OKExChainApp) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "log" | ||
|
|
||
| "github.com/cosmos/cosmos-sdk/client/flags" | ||
| "github.com/cosmos/cosmos-sdk/server" | ||
| "github.com/spf13/cobra" | ||
| "github.com/spf13/viper" | ||
| cfg "github.com/tendermint/tendermint/config" | ||
| "github.com/tendermint/tendermint/node" | ||
| sm "github.com/tendermint/tendermint/state" | ||
| "github.com/tendermint/tendermint/store" | ||
| dbm "github.com/tendermint/tm-db" | ||
| ) | ||
|
|
||
| func pruningCmd(ctx *server.Context) *cobra.Command { | ||
| cmd := &cobra.Command{ | ||
| Use: "pruning", | ||
| Short: "Pruning blocks", | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| config := ctx.Config | ||
| config.SetRoot(viper.GetString(flags.FlagHome)) | ||
| log.Println("--------- pruning start ---------") | ||
| blockStoreDB, stateDB, _, err := initDBs(config, node.DefaultDBProvider) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| blockStore := store.NewBlockStore(blockStoreDB) | ||
| baseHeight := blockStore.Base() | ||
| size := blockStore.Size() | ||
| retainHeight := baseHeight + size - 2 | ||
| log.Printf("baseHeight:%d, size:%d, retainHeight:%d\n", baseHeight, size, retainHeight) | ||
|
|
||
| pruneBlocks(blockStore, stateDB, retainHeight) | ||
|
|
||
| log.Println("--------- pruning end ---------") | ||
| return nil | ||
| }, | ||
| } | ||
|
|
||
| return cmd | ||
| } | ||
|
|
||
| func initDBs(config *cfg.Config, dbProvider node.DBProvider) (blockStoreDB, stateDB, appDB dbm.DB, err error) { | ||
| blockStoreDB, err = dbProvider(&node.DBContext{"blockstore", config}) | ||
| if err != nil { | ||
| return | ||
| } | ||
|
|
||
| stateDB, err = dbProvider(&node.DBContext{"state", config}) | ||
| if err != nil { | ||
| return | ||
| } | ||
|
|
||
| appDB, err = dbProvider(&node.DBContext{"application", config}) | ||
| if err != nil { | ||
| return | ||
| } | ||
|
|
||
| return | ||
| } | ||
|
|
||
| func pruneBlocks(blockStore *store.BlockStore, stateDB dbm.DB, retainHeight int64) { | ||
| base := blockStore.Base() | ||
| if retainHeight <= base { | ||
| return | ||
| } | ||
| pruned, err := blockStore.PruneBlocks(retainHeight) | ||
| if err != nil { | ||
| panic(fmt.Errorf("failed to prune block store: %w", err)) | ||
| } | ||
| err = sm.PruneStates(stateDB, base, retainHeight) | ||
| if err != nil { | ||
| panic(fmt.Errorf("failed to prune state database: %w", err)) | ||
| } | ||
|
|
||
| log.Printf("pruned blocks: %d, retainHeight: %d\n", pruned, retainHeight) | ||
| log.Printf("block store base: %d, block store size: %d\n", blockStore.Base(), blockStore.Size()) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters