From 2852d9e3cb685a559586efbcba295716f267f2af Mon Sep 17 00:00:00 2001 From: KamiD <44460798+KamiD@users.noreply.github.com> Date: Mon, 7 Jun 2021 10:23:48 +0800 Subject: [PATCH] =?UTF-8?q?Merge=20PR=20=EF=BC=9Aadd=20state=20lru=20cache?= =?UTF-8?q?=20to=20optimize=20eth=5Fcall=20(#898)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add 1KW LRU cache to optimize eth_call * optimize code --- app/rpc/namespaces/eth/state/lru.go | 62 +++++++++++++++++++++++++++++ x/evm/watcher/querier.go | 31 ++++++++++----- x/evm/watcher/types.go | 60 ++++++++++++++++++++++++++-- x/evm/watcher/watcher.go | 5 +++ 4 files changed, 144 insertions(+), 14 deletions(-) create mode 100644 app/rpc/namespaces/eth/state/lru.go diff --git a/app/rpc/namespaces/eth/state/lru.go b/app/rpc/namespaces/eth/state/lru.go new file mode 100644 index 0000000000..a2025a052d --- /dev/null +++ b/app/rpc/namespaces/eth/state/lru.go @@ -0,0 +1,62 @@ +package state + +import ( + "errors" + "sync" + + "github.com/spf13/viper" + + "github.com/ethereum/go-ethereum/common" + + lru "github.com/hashicorp/golang-lru" +) + +//the default lru cache size is 1kw, that means the max memory size we needs is (32 + 32 + 4) * 10000000, about 700MB +var ( + defaultLruSize int = 10000000 + gStateLru *lru.Cache = nil + once sync.Once +) + +//redefine fast-query to avoid cycle package import +const FlagFastQuery = "fast-query" + +func isWatcherEnabled() bool { + return viper.GetBool(FlagFastQuery) +} + +func InstanceOfStateLru() *lru.Cache { + once.Do(func() { + if isWatcherEnabled() { + var e error = nil + gStateLru, e = lru.New(defaultLruSize) + if e != nil { + panic(errors.New("Failed to call InstanceOfStateLru cause :" + e.Error())) + } + } + }) + return gStateLru +} + +func GetStateFromLru(key common.Hash) []byte { + cache := InstanceOfStateLru() + if cache == nil { + return nil + } + value, ok := cache.Get(key) + if ok { + ret, ok := value.([]byte) + if ok { + return ret + } + } + return nil +} + +func SetStateToLru(key common.Hash, value []byte) { + cache := InstanceOfStateLru() + if cache == nil { + return + } + cache.Add(key, value) +} diff --git a/x/evm/watcher/querier.go b/x/evm/watcher/querier.go index 11f764e962..777c45011c 100644 --- a/x/evm/watcher/querier.go +++ b/x/evm/watcher/querier.go @@ -6,6 +6,8 @@ import ( "errors" "strconv" + "github.com/okex/exchain/app/rpc/namespaces/eth/state" + lru "github.com/hashicorp/golang-lru" sdk "github.com/cosmos/cosmos-sdk/types" @@ -14,7 +16,6 @@ import ( rpctypes "github.com/okex/exchain/app/rpc/types" "github.com/okex/exchain/app/types" evmtypes "github.com/okex/exchain/x/evm/types" - "github.com/status-im/keycard-go/hexutils" ) const MsgFunctionDisable = "fast query function has been disabled" @@ -285,37 +286,45 @@ func (q Querier) DeleteAccountFromRdb(addr sdk.AccAddress) { } func (q Querier) MustGetState(addr common.Address, key []byte) ([]byte, error) { - b, e := q.GetState(addr, key) + orgKey := GetMsgStateKey(addr, key) + realKey := common.BytesToHash(orgKey) + data := state.GetStateFromLru(realKey) + if data != nil { + return data, nil + } + b, e := q.GetState(orgKey) if e != nil { - b, e = q.GetStateFromRdb(addr, key) + b, e = q.GetStateFromRdb(orgKey) } else { q.DeleteStateFromRdb(addr, key) } + if e == nil { + state.SetStateToLru(realKey, b) + } return b, e } -func (q Querier) GetState(addr common.Address, key []byte) ([]byte, error) { +func (q Querier) GetState(key []byte) ([]byte, error) { if !q.enabled() { return nil, errors.New(MsgFunctionDisable) } - b, e := q.store.Get(GetMsgStateKey(addr, key)) + b, e := q.store.Get(key) if e != nil { return nil, e } - ret := hexutils.HexToBytes(string(b)) - return ret, nil + return b, nil } -func (q Querier) GetStateFromRdb(addr common.Address, key []byte) ([]byte, error) { +func (q Querier) GetStateFromRdb(key []byte) ([]byte, error) { if !q.enabled() { return nil, errors.New(MsgFunctionDisable) } - b, e := q.store.Get(append(prefixRpcDb, GetMsgStateKey(addr, key)...)) + b, e := q.store.Get(append(prefixRpcDb, key...)) if e != nil { return nil, e } - ret := hexutils.HexToBytes(string(b)) - return ret, nil + + return b, nil } func (q Querier) DeleteStateFromRdb(addr common.Address, key []byte) { diff --git a/x/evm/watcher/types.go b/x/evm/watcher/types.go index 63cadba64d..c577809058 100644 --- a/x/evm/watcher/types.go +++ b/x/evm/watcher/types.go @@ -42,9 +42,15 @@ var ( TransactionFailed = uint32(0) ) +const ( + TypeOthers = uint32(1) + TypeState = uint32(2) +) + type WatchMessage interface { GetKey() []byte GetValue() string + GetType() uint32 } type MsgEthTx struct { @@ -52,6 +58,10 @@ type MsgEthTx struct { JsonEthTx string } +func (m MsgEthTx) GetType() uint32 { + return TypeOthers +} + func NewMsgEthTx(tx *types.MsgEthereumTx, txHash, blockHash common.Hash, height, index uint64) *MsgEthTx { ethTx, e := rpctypes.NewTransaction(tx, txHash, blockHash, height, index) if e != nil { @@ -81,6 +91,10 @@ type MsgCode struct { Code string } +func (m MsgCode) GetType() uint32 { + return TypeOthers +} + type CodeInfo struct { Height uint64 `json:"height"` Code string `json:"code"` @@ -114,6 +128,10 @@ type MsgCodeByHash struct { Code string } +func (m MsgCodeByHash) GetType() uint32 { + return TypeOthers +} + func NewMsgCodeByHash(hash []byte, code []byte) *MsgCodeByHash { return &MsgCodeByHash{ Key: hash, @@ -134,6 +152,10 @@ type MsgTransactionReceipt struct { receipt string } +func (m MsgTransactionReceipt) GetType() uint32 { + return TypeOthers +} + type TransactionReceipt struct { Status hexutil.Uint64 `json:"status"` CumulativeGasUsed hexutil.Uint64 `json:"cumulativeGasUsed"` @@ -191,6 +213,10 @@ type MsgBlock struct { block string } +func (m MsgBlock) GetType() uint32 { + return TypeOthers +} + // A BlockNonce is a 64-bit hash which proves (combined with the // mix-hash) that a sufficient amount of computation has been carried // out on a block. @@ -284,6 +310,10 @@ type MsgBlockInfo struct { hash string } +func (b MsgBlockInfo) GetType() uint32 { + return TypeOthers +} + func NewMsgBlockInfo(height uint64, blockHash common.Hash) *MsgBlockInfo { return &MsgBlockInfo{ height: []byte(strconv.Itoa(int(height))), @@ -303,6 +333,10 @@ type MsgLatestHeight struct { height string } +func (b MsgLatestHeight) GetType() uint32 { + return TypeOthers +} + func NewMsgLatestHeight(height uint64) *MsgLatestHeight { return &MsgLatestHeight{ height: strconv.Itoa(int(height)), @@ -322,6 +356,10 @@ type MsgAccount struct { accountValue string } +func (msgAccount *MsgAccount) GetType() uint32 { + return TypeOthers +} + func NewMsgAccount(acc auth.Account) *MsgAccount { jsonAcc, err := json.Marshal(acc) if err != nil { @@ -348,14 +386,18 @@ func (msgAccount *MsgAccount) GetValue() string { type MsgState struct { addr common.Address key []byte - value string + value []byte +} + +func (msgState *MsgState) GetType() uint32 { + return TypeState } func NewMsgState(addr common.Address, key, value []byte) *MsgState { return &MsgState{ addr: addr, key: key, - value: hexutils.BytesToHex(value), + value: value, } } @@ -374,13 +416,17 @@ func (msgState *MsgState) GetKey() []byte { } func (msgState *MsgState) GetValue() string { - return msgState.value + return string(msgState.value) } type MsgParams struct { types.Params } +func (msgParams *MsgParams) GetType() uint32 { + return TypeOthers +} + func NewMsgParams(params types.Params) *MsgParams { return &MsgParams{ params, @@ -403,6 +449,10 @@ type MsgContractBlockedListItem struct { addr sdk.AccAddress } +func (msgItem *MsgContractBlockedListItem) GetType() uint32 { + return TypeOthers +} + func NewMsgContractBlockedListItem(addr sdk.AccAddress) *MsgContractBlockedListItem { return &MsgContractBlockedListItem{ addr: addr, @@ -421,6 +471,10 @@ type MsgContractDeploymentWhitelistItem struct { addr sdk.AccAddress } +func (msgItem *MsgContractDeploymentWhitelistItem) GetType() uint32 { + return TypeOthers +} + func NewMsgContractDeploymentWhitelistItem(addr sdk.AccAddress) *MsgContractDeploymentWhitelistItem { return &MsgContractDeploymentWhitelistItem{ addr: addr, diff --git a/x/evm/watcher/watcher.go b/x/evm/watcher/watcher.go index 45a76904cb..fbfb8d5f15 100644 --- a/x/evm/watcher/watcher.go +++ b/x/evm/watcher/watcher.go @@ -3,6 +3,8 @@ package watcher import ( "math/big" + "github.com/okex/exchain/app/rpc/namespaces/eth/state" + sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/auth" @@ -288,6 +290,9 @@ func (w *Watcher) Commit() { go func() { for _, b := range batch { w.store.Set(b.GetKey(), []byte(b.GetValue())) + if b.GetType() == TypeState { + state.SetStateToLru(common.BytesToHash(b.GetKey()), []byte(b.GetValue())) + } } }() }