Merge PR: init commit and merge confilict (#2731)

when repair-state we judge the fast-storage flag by data

add feesplit module

reserve --iavl-enable-fast-storage flag for repair-state

merge repair-state we judge... and deal with confict

rm NeedLongTimeToUpgrade use the IsFastStorgeStrategy universal

Co-authored-by: xiangjianmeng <[email protected]>

app/utils/appstatus/fast_storage.go

@@ -0,0 +1,85 @@
+package appstatus
+
+import (
+	"fmt"
+	"path/filepath"
+
+	bam "github.com/okex/exchain/libs/cosmos-sdk/baseapp"
+	"github.com/okex/exchain/libs/cosmos-sdk/client/flags"
+	"github.com/okex/exchain/libs/cosmos-sdk/store/mpt"
+	sdk "github.com/okex/exchain/libs/cosmos-sdk/types"
+	"github.com/okex/exchain/libs/cosmos-sdk/x/auth"
+	capabilitytypes "github.com/okex/exchain/libs/cosmos-sdk/x/capability/types"
+	"github.com/okex/exchain/libs/cosmos-sdk/x/mint"
+	"github.com/okex/exchain/libs/cosmos-sdk/x/params"
+	"github.com/okex/exchain/libs/cosmos-sdk/x/supply"
+	"github.com/okex/exchain/libs/cosmos-sdk/x/upgrade"
+	"github.com/okex/exchain/libs/iavl"
+	ibctransfertypes "github.com/okex/exchain/libs/ibc-go/modules/apps/transfer/types"
+	ibchost "github.com/okex/exchain/libs/ibc-go/modules/core/24-host"
+	dbm "github.com/okex/exchain/libs/tm-db"
+	"github.com/okex/exchain/x/ammswap"
+	dex "github.com/okex/exchain/x/dex/types"
+	distr "github.com/okex/exchain/x/distribution"
+	"github.com/okex/exchain/x/erc20"
+	"github.com/okex/exchain/x/evidence"
+	"github.com/okex/exchain/x/evm"
+	"github.com/okex/exchain/x/farm"
+	"github.com/okex/exchain/x/feesplit"
+	"github.com/okex/exchain/x/gov"
+	"github.com/okex/exchain/x/order"
+	"github.com/okex/exchain/x/slashing"
+	staking "github.com/okex/exchain/x/staking/types"
+	token "github.com/okex/exchain/x/token/types"
+	"github.com/okex/exchain/x/wasm"
+	"github.com/spf13/viper"
+)
+
+const (
+	applicationDB = "application"
+	dbFolder      = "data"
+)
+
+func GetAllStoreKeys() []string {
+	return []string{
+		bam.MainStoreKey, auth.StoreKey, staking.StoreKey,
+		supply.StoreKey, mint.StoreKey, distr.StoreKey, slashing.StoreKey,
+		gov.StoreKey, params.StoreKey, upgrade.StoreKey, evidence.StoreKey,
+		evm.StoreKey, token.StoreKey, token.KeyLock, dex.StoreKey, dex.TokenPairStoreKey,
+		order.OrderStoreKey, ammswap.StoreKey, farm.StoreKey, ibctransfertypes.StoreKey, capabilitytypes.StoreKey,
+		ibchost.StoreKey,
+		erc20.StoreKey,
+		mpt.StoreKey,
+		wasm.StoreKey,
+		feesplit.StoreKey,
+	}
+}
+
+func IsFastStorageStrategy() bool {
+	return checkFastStorageStrategy(GetAllStoreKeys())
+}
+
+func checkFastStorageStrategy(storeKeys []string) bool {
+	home := viper.GetString(flags.FlagHome)
+	dataDir := filepath.Join(home, dbFolder)
+	db, err := sdk.NewDB(applicationDB, dataDir)
+	if err != nil {
+		panic(err)
+	}
+	defer db.Close()
+
+	for _, v := range storeKeys {
+		if !isFss(db, v) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func isFss(db dbm.DB, storeKey string) bool {
+	prefix := fmt.Sprintf("s/k:%s/", storeKey)
+	prefixDB := dbm.NewPrefixDB(db, []byte(prefix))
+
+	return iavl.IsFastStorageStrategy(prefixDB)
+}

app/utils/sanity/start.go

@@ -1,10 +1,14 @@
 package sanity
 
 import (
+	"fmt"
+
 	apptype "github.com/okex/exchain/app/types"
+	"github.com/okex/exchain/app/utils/appstatus"
 	"github.com/okex/exchain/libs/cosmos-sdk/server"
 	cosmost "github.com/okex/exchain/libs/cosmos-sdk/store/types"
 	sdk "github.com/okex/exchain/libs/cosmos-sdk/types"
+	"github.com/okex/exchain/libs/iavl"
 	"github.com/okex/exchain/libs/tendermint/consensus"
 	"github.com/okex/exchain/libs/tendermint/state"
 	"github.com/okex/exchain/libs/tendermint/types"
@@ -53,42 +57,49 @@ import (
 var (
 	startDependentElems = []dependentPair{
 		{ // if infura.FlagEnable=true , watcher.FlagFastQuery must be set to true
-			config:       boolItem{name: infura.FlagEnable, value: true},
-			reliedConfig: boolItem{name: watcher.FlagFastQuery, value: true},
+			config:       boolItem{name: infura.FlagEnable, expect: true},
+			reliedConfig: boolItem{name: watcher.FlagFastQuery, expect: true},
 		},
 	}
 	// conflicts flags
 	startConflictElems = []conflictPair{
 		// --fast-query      conflict with --pruning=nothing
 		{
-			configA: boolItem{name: watcher.FlagFastQuery, value: true},
-			configB: stringItem{name: server.FlagPruning, value: cosmost.PruningOptionNothing},
+			configA: boolItem{name: watcher.FlagFastQuery, expect: true},
+			configB: stringItem{name: server.FlagPruning, expect: cosmost.PruningOptionNothing},
 		},
 		// --enable-preruntx conflict with --download-delta
 		{
-			configA: boolItem{name: consensus.EnablePrerunTx, value: true},
-			configB: boolItem{name: types.FlagDownloadDDS, value: true},
+			configA: boolItem{name: consensus.EnablePrerunTx, expect: true},
+			configB: boolItem{name: types.FlagDownloadDDS, expect: true},
 		},
 		// --multi-cache conflict with --download-delta
 		{
-			configA: boolItem{name: sdk.FlagMultiCache, value: true},
-			configB: boolItem{name: types.FlagDownloadDDS, value: true},
+			configA: boolItem{name: sdk.FlagMultiCache, expect: true},
+			configB: boolItem{name: types.FlagDownloadDDS, expect: true},
 		},
 		{
-			configA: stringItem{name: apptype.FlagNodeMode, value: string(apptype.RpcNode)},
-			configB: stringItem{name: server.FlagPruning, value: cosmost.PruningOptionNothing},
+			configA: stringItem{name: apptype.FlagNodeMode, expect: string(apptype.RpcNode)},
+			configB: stringItem{name: server.FlagPruning, expect: cosmost.PruningOptionNothing},
 		},
 		// --node-mode=archive(--pruning=nothing) conflicts with --fast-query
 		{
-			configA: stringItem{name: apptype.FlagNodeMode, value: string(apptype.ArchiveNode)},
-			configB: boolItem{name: watcher.FlagFastQuery, value: true},
+			configA: stringItem{name: apptype.FlagNodeMode, expect: string(apptype.ArchiveNode)},
+			configB: boolItem{name: watcher.FlagFastQuery, expect: true},
+		},
+		{
+			configA: boolItem{name: iavl.FlagIavlEnableFastStorage, expect: true},
+			configB: funcItem{name: "Upgraded to fast IAVL", expect: false, f: appstatus.IsFastStorageStrategy},
+			tips: fmt.Sprintf("Upgrade to IAVL fast storage may take several hours, "+
+				"you can use exchaind fss create command to upgrade, or unset --%v", iavl.FlagIavlEnableFastStorage),
 		},
 	}
 
-	checkRangeItems = []rangeItem{{
-		enumRange: []int{int(state.DeliverTxsExecModeSerial), state.DeliverTxsExecModeParallel},
-		name:      state.FlagDeliverTxsExecMode,
-	},
+	checkRangeItems = []rangeItem{
+		{
+			enumRange: []int{int(state.DeliverTxsExecModeSerial), state.DeliverTxsExecModeParallel},
+			name:      state.FlagDeliverTxsExecMode,
+		},
 	}
 )
 

app/utils/sanity/type.go

@@ -15,61 +15,64 @@ const (
 type item interface {
 	// label: get item's name
 	label() string
-	// check: whether the userSetting value is equal to the value
+	// check: whether the userSetting value is equal to expect value
 	check() bool
 	// verbose: show the readable flag
 	verbose() string
 }
 
 type boolItem struct {
-	name  string
-	value bool
+	name   string
+	expect bool
 }
 
 func (b boolItem) label() string {
 	return b.name
 }
 
 func (b boolItem) check() bool {
-	return viper.GetBool(b.label()) == b.value
+	return viper.GetBool(b.label()) == b.expect
 }
 
 func (b boolItem) verbose() string {
-	return fmt.Sprintf("--%v=%v", b.name, b.value)
+	return fmt.Sprintf("--%v=%v", b.name, b.expect)
 }
 
-type intItem struct {
-	name  string
-	value int
+type stringItem struct {
+	name   string
+	expect string
 }
 
-func (i intItem) label() string {
-	return i.name
+func (s stringItem) label() string {
+	return s.name
 }
 
-func (i intItem) check() bool {
-	return viper.GetInt(i.label()) == i.value
+func (s stringItem) check() bool {
+	return strings.ToLower(viper.GetString(s.label())) == s.expect
 }
 
-func (i intItem) verbose() string {
-	return fmt.Sprintf("--%v=%v", i.name, i.value)
+func (s stringItem) verbose() string {
+	return fmt.Sprintf("--%v=%v", s.name, s.expect)
 }
 
-type stringItem struct {
-	name  string
-	value string
+type funcItem struct {
+	name   string
+	expect bool
+	actual bool
+	f      func() bool
 }
 
-func (s stringItem) label() string {
-	return s.name
+func (f funcItem) label() string {
+	return f.name
 }
 
-func (s stringItem) check() bool {
-	return strings.ToLower(viper.GetString(s.label())) == s.value
+func (f funcItem) check() bool {
+	f.actual = f.f()
+	return f.actual == f.expect
 }
 
-func (s stringItem) verbose() string {
-	return fmt.Sprintf("--%v=%v", s.name, s.value)
+func (f funcItem) verbose() string {
+	return fmt.Sprintf("%v=%v", f.name, f.actual)
 }
 
 type dependentPair struct {
@@ -90,14 +93,19 @@ func (cp *dependentPair) check() error {
 type conflictPair struct {
 	configA item
 	configB item
+	tips    string
 }
 
 // checkConflict: check configA vs configB
-// and the value is equal to the conflicts value then complain it
+// if both configA and configB are got expect values
+// then complain it. if there is a custom tips use it.
 func (cp *conflictPair) check() error {
 	if cp.configA.check() &&
 		cp.configB.check() {
-		return fmt.Errorf(" %v conflict with %v", cp.configA.verbose(), cp.configB.verbose())
+		if cp.tips == "" {
+			return fmt.Errorf(" %v conflict with %v", cp.configA.verbose(), cp.configB.verbose())
+		}
+		return fmt.Errorf(cp.tips)
 	}
 
 	return nil

app/utils/sanity/type_test.go

@@ -163,13 +163,13 @@ func Test_conflictPair_checkConflict(t *testing.T) {
 		wantErr bool
 	}{
 		{name: "1. bool item and bool item both true",
-			fields: fields{configA: boolItem{name: "b1", value: true}, configB: boolItem{name: "b2", value: true}},
+			fields: fields{configA: boolItem{name: "b1", expect: true}, configB: boolItem{name: "b2", expect: true}},
 			args:   args{cmd: getCommandBool()}, wantErr: true},
 		{name: "2. bool item and bool item true vs false",
-			fields: fields{configA: boolItem{name: "b1", value: true}, configB: boolItem{name: "b3", value: false}},
+			fields: fields{configA: boolItem{name: "b1", expect: true}, configB: boolItem{name: "b3", expect: false}},
 			args:   args{cmd: getCommandBoolDiff()}, wantErr: true},
 		{name: "3. bool item and string item",
-			fields: fields{configA: boolItem{name: "b1", value: true}, configB: stringItem{name: "s1", value: "conflict"}},
+			fields: fields{configA: boolItem{name: "b1", expect: true}, configB: stringItem{name: "s1", expect: "conflict"}},
 			args:   args{cmd: getCommandBoolString()}, wantErr: true},
 	}
 	for _, tt := range tests {
@@ -198,13 +198,13 @@ func Test_dependentPair_check(t *testing.T) {
 		wantErr bool
 	}{
 		{name: "1. b1=true, b2=true, correct",
-			fields: dependentPair{config: boolItem{name: "b1", value: true}, reliedConfig: boolItem{name: "b2", value: true}},
+			fields: dependentPair{config: boolItem{name: "b1", expect: true}, reliedConfig: boolItem{name: "b2", expect: true}},
 			args:   args{cmd: getCommandBool()}, wantErr: false},
 		{name: "2. b1=true,b2=false, need error",
-			fields: dependentPair{config: boolItem{name: "b1", value: true}, reliedConfig: boolItem{name: "b2", value: false}},
+			fields: dependentPair{config: boolItem{name: "b1", expect: true}, reliedConfig: boolItem{name: "b2", expect: false}},
 			args:   args{cmd: getCommandBool()}, wantErr: true},
 		{name: "2. b1=false, no error",
-			fields: dependentPair{config: boolItem{name: "b1", value: false}, reliedConfig: boolItem{name: "b2", value: false}},
+			fields: dependentPair{config: boolItem{name: "b1", expect: false}, reliedConfig: boolItem{name: "b2", expect: false}},
 			args:   args{cmd: getCommandBool()}, wantErr: false},
 	}
 	for _, tt := range tests {

cmd/exchaind/fss/create.go

@@ -3,12 +3,12 @@ package fss
 import (
 	"fmt"
 	"log"
+	"path/filepath"
 
+	"github.com/okex/exchain/app/utils/appstatus"
 	"github.com/okex/exchain/cmd/exchaind/base"
-	"github.com/okex/exchain/libs/cosmos-sdk/x/auth"
 	"github.com/okex/exchain/libs/iavl"
 	dbm "github.com/okex/exchain/libs/tm-db"
-	"github.com/okex/exchain/x/evm"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -23,7 +23,7 @@ It will take long based on the original database size.
 When the create lunched, it will show Upgrade to Fast IAVL...`,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		iavl.SetEnableFastStorage(true)
-		storeKeys := getStoreKeys()
+		storeKeys := appstatus.GetAllStoreKeys()
 		outputModules(storeKeys)
 
 		return createIndex(storeKeys)
@@ -34,13 +34,6 @@ func init() {
 	fssCmd.AddCommand(createCmd)
 }
 
-func getStoreKeys() []string {
-	return []string{
-		auth.StoreKey,
-		evm.StoreKey,
-	}
-}
-
 func outputModules(storeKeys []string) {
 	if iavl.OutputModules == nil {
 		iavl.OutputModules = make(map[string]int, len(storeKeys))
@@ -53,7 +46,7 @@ func outputModules(storeKeys []string) {
 func createIndex(storeKeys []string) error {
 	dataDir := viper.GetString(flagDataDir)
 	dbBackend := viper.GetString(flagDBBackend)
-	db, err := base.OpenDB(dataDir+base.AppDBName, dbm.BackendType(dbBackend))
+	db, err := base.OpenDB(filepath.Join(dataDir, base.AppDBName), dbm.BackendType(dbBackend))
 	if err != nil {
 		return fmt.Errorf("error opening dir %v backend %v DB: %w", dataDir, dbBackend, err)
 	}

cmd/exchaind/repair_data.go

@@ -4,6 +4,7 @@ import (
 	"log"
 
 	"github.com/okex/exchain/app"
+	"github.com/okex/exchain/app/utils/appstatus"
 	"github.com/okex/exchain/libs/cosmos-sdk/server"
 	"github.com/okex/exchain/libs/cosmos-sdk/store/flatkv"
 	sdk "github.com/okex/exchain/libs/cosmos-sdk/types"
@@ -37,13 +38,13 @@ func repairStateCmd(ctx *server.Context) *cobra.Command {
 	cmd.Flags().BoolVar(&types2.TrieUseCompositeKey, types2.FlagTrieUseCompositeKey, true, "Use composite key to store contract state")
 	cmd.Flags().Int(sm.FlagDeliverTxsExecMode, 0, "execution mode for deliver txs, (0:serial[default], 1:deprecated, 2:parallel)")
 	cmd.Flags().Bool(tmiavl.FlagIavlEnableFastStorage, false, "Enable fast storage")
-	cmd.Flags().Int(tmiavl.FlagIavlFastStorageCacheSize, 100000, "Max size of iavl fast storage cache")
 	cmd.Flags().String(sdk.FlagDBBackend, tmtypes.DBBackend, "Database backend: goleveldb | rocksdb")
 
 	return cmd
 }
 
 func setExternalPackageValue() {
-	tmiavl.SetEnableFastStorage(viper.GetBool(tmiavl.FlagIavlEnableFastStorage))
-	tmiavl.SetFastNodeCacheSize(viper.GetInt(tmiavl.FlagIavlFastStorageCacheSize))
+	enableFastStorage := viper.GetBool(tmiavl.FlagIavlEnableFastStorage) ||
+		appstatus.IsFastStorageStrategy()
+	tmiavl.SetEnableFastStorage(enableFastStorage)
 }