diff --git a/app/app.go b/app/app.go index 42bcd4014f..6ebe1c8dbb 100644 --- a/app/app.go +++ b/app/app.go @@ -31,10 +31,14 @@ import ( "cosmossdk.io/core/appmodule" runtimeservices "github.com/cosmos/cosmos-sdk/runtime/services" "github.com/cosmos/cosmos-sdk/server" + grpc1 "github.com/cosmos/gogoproto/grpc" "github.com/cosmos/gogoproto/proto" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" "github.com/gorilla/mux" "github.com/spf13/cast" + "github.com/spf13/viper" "cosmossdk.io/log" abci "github.com/cometbft/cometbft/abci/types" @@ -132,6 +136,7 @@ import ( "github.com/evmos/ethermint/app/ante" "github.com/evmos/ethermint/encoding" "github.com/evmos/ethermint/ethereum/eip712" + srv "github.com/evmos/ethermint/server" srvconfig "github.com/evmos/ethermint/server/config" srvflags "github.com/evmos/ethermint/server/flags" ethermint "github.com/evmos/ethermint/types" @@ -249,6 +254,50 @@ type EthermintApp struct { configurator module.Configurator } +func BackupQueryClients[T any]( + appOpts servertypes.AppOptions, + interfaceRegistry types.InterfaceRegistry, + queryClientFn func(grpc1.ClientConn) T, +) map[[2]int]T { + backupQueryClients := make(map[[2]int]T) + if v, ok := appOpts.(*viper.Viper); ok { + cfg, err := srvconfig.GetConfig(v) + if err == nil { + maxSendMsgSize := cfg.GRPC.MaxSendMsgSize + if maxSendMsgSize == 0 { + maxSendMsgSize = config.DefaultGRPCMaxSendMsgSize + } + + maxRecvMsgSize := cfg.GRPC.MaxRecvMsgSize + if maxRecvMsgSize == 0 { + maxRecvMsgSize = config.DefaultGRPCMaxRecvMsgSize + } + + for k, address := range cfg.JSONRPC.BackupGRPCBlockAddressBlockRange { + grpcAddr, err := srv.ParseGRPCAddress(address) + if err != nil { + continue + } + + conn, err := grpc.Dial( + grpcAddr, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultCallOptions( + grpc.ForceCodec(codec.NewProtoCodec(interfaceRegistry).GRPCCodec()), + grpc.MaxCallRecvMsgSize(maxRecvMsgSize), + grpc.MaxCallSendMsgSize(maxSendMsgSize), + ), + ) + if err != nil { + continue + } + backupQueryClients[k] = queryClientFn(conn) + } + } + } + return backupQueryClients +} + // NewEthermintApp returns a reference to a new initialized Ethermint application. func NewEthermintApp( logger log.Logger, @@ -485,6 +534,7 @@ func NewEthermintApp( authtypes.NewModuleAddress(govtypes.ModuleName), keys[feemarkettypes.StoreKey], feeMarketSs, + BackupQueryClients(appOpts, interfaceRegistry, feemarkettypes.NewQueryClient), ) // Set authority to x/gov module account to only expect the module account to update params @@ -496,6 +546,7 @@ func NewEthermintApp( tracer, evmSs, nil, + BackupQueryClients(appOpts, interfaceRegistry, evmtypes.NewQueryClient), ) // register the proposal types diff --git a/server/config/config.go b/server/config/config.go index 8e9167719b..6acfac8a3e 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -16,6 +16,7 @@ package config import ( + "encoding/json" "errors" "fmt" "path" @@ -179,6 +180,8 @@ type JSONRPCConfig struct { FixRevertGasRefundHeight int64 `mapstructure:"fix-revert-gas-refund-height"` // ReturnDataLimit defines maximum number of bytes returned from `eth_call` or similar invocations ReturnDataLimit int64 `mapstructure:"return-data-limit"` + // A list of grpc address with block range + BackupGRPCBlockAddressBlockRange map[[2]int]string `mapstructure:"backup-grpc-address-block-range"` } // TLSConfig defines the certificate and matching private key for the server. @@ -277,26 +280,27 @@ func GetAPINamespaces() []string { // DefaultJSONRPCConfig returns an EVM config with the JSON-RPC API enabled by default func DefaultJSONRPCConfig() *JSONRPCConfig { return &JSONRPCConfig{ - Enable: true, - API: GetDefaultAPINamespaces(), - Address: DefaultJSONRPCAddress, - WsAddress: DefaultJSONRPCWsAddress, - GasCap: DefaultGasCap, - EVMTimeout: DefaultEVMTimeout, - TxFeeCap: DefaultTxFeeCap, - FilterCap: DefaultFilterCap, - FeeHistoryCap: DefaultFeeHistoryCap, - BlockRangeCap: DefaultBlockRangeCap, - LogsCap: DefaultLogsCap, - HTTPTimeout: DefaultHTTPTimeout, - HTTPIdleTimeout: DefaultHTTPIdleTimeout, - AllowUnprotectedTxs: DefaultAllowUnprotectedTxs, - MaxOpenConnections: DefaultMaxOpenConnections, - EnableIndexer: false, - AllowIndexerGap: true, - MetricsAddress: DefaultJSONRPCMetricsAddress, - FixRevertGasRefundHeight: DefaultFixRevertGasRefundHeight, - ReturnDataLimit: DefaultReturnDataLimit, + Enable: true, + API: GetDefaultAPINamespaces(), + Address: DefaultJSONRPCAddress, + WsAddress: DefaultJSONRPCWsAddress, + GasCap: DefaultGasCap, + EVMTimeout: DefaultEVMTimeout, + TxFeeCap: DefaultTxFeeCap, + FilterCap: DefaultFilterCap, + FeeHistoryCap: DefaultFeeHistoryCap, + BlockRangeCap: DefaultBlockRangeCap, + LogsCap: DefaultLogsCap, + HTTPTimeout: DefaultHTTPTimeout, + HTTPIdleTimeout: DefaultHTTPIdleTimeout, + AllowUnprotectedTxs: DefaultAllowUnprotectedTxs, + MaxOpenConnections: DefaultMaxOpenConnections, + EnableIndexer: false, + AllowIndexerGap: true, + MetricsAddress: DefaultJSONRPCMetricsAddress, + FixRevertGasRefundHeight: DefaultFixRevertGasRefundHeight, + ReturnDataLimit: DefaultReturnDataLimit, + BackupGRPCBlockAddressBlockRange: make(map[[2]int]string), } } @@ -403,6 +407,18 @@ func GetConfig(v *viper.Viper) (Config, error) { return Config{}, err } + data := make(map[string][2]int) + raw := v.GetString("json-rpc.backup-grpc-address-block-range") + if len(raw) > 0 { + err = json.Unmarshal([]byte(raw), &data) + if err != nil { + return Config{}, err + } + } + backupGRPCBlockAddressBlockRange := make(map[[2]int]string) + for k, v := range data { + backupGRPCBlockAddressBlockRange[v] = k + } return Config{ Config: cfg, EVM: EVMConfig{ @@ -412,25 +428,26 @@ func GetConfig(v *viper.Viper) (Config, error) { BlockSTMWorkers: v.GetInt("evm.block-stm-workers"), }, JSONRPC: JSONRPCConfig{ - Enable: v.GetBool("json-rpc.enable"), - API: v.GetStringSlice("json-rpc.api"), - Address: v.GetString("json-rpc.address"), - WsAddress: v.GetString("json-rpc.ws-address"), - GasCap: v.GetUint64("json-rpc.gas-cap"), - FilterCap: v.GetInt32("json-rpc.filter-cap"), - FeeHistoryCap: v.GetInt32("json-rpc.feehistory-cap"), - TxFeeCap: v.GetFloat64("json-rpc.txfee-cap"), - EVMTimeout: v.GetDuration("json-rpc.evm-timeout"), - LogsCap: v.GetInt32("json-rpc.logs-cap"), - BlockRangeCap: v.GetInt32("json-rpc.block-range-cap"), - HTTPTimeout: v.GetDuration("json-rpc.http-timeout"), - HTTPIdleTimeout: v.GetDuration("json-rpc.http-idle-timeout"), - MaxOpenConnections: v.GetInt("json-rpc.max-open-connections"), - EnableIndexer: v.GetBool("json-rpc.enable-indexer"), - AllowIndexerGap: v.GetBool("json-rpc.allow-indexer-gap"), - MetricsAddress: v.GetString("json-rpc.metrics-address"), - FixRevertGasRefundHeight: v.GetInt64("json-rpc.fix-revert-gas-refund-height"), - ReturnDataLimit: v.GetInt64("json-rpc.return-data-limit"), + Enable: v.GetBool("json-rpc.enable"), + API: v.GetStringSlice("json-rpc.api"), + Address: v.GetString("json-rpc.address"), + WsAddress: v.GetString("json-rpc.ws-address"), + GasCap: v.GetUint64("json-rpc.gas-cap"), + FilterCap: v.GetInt32("json-rpc.filter-cap"), + FeeHistoryCap: v.GetInt32("json-rpc.feehistory-cap"), + TxFeeCap: v.GetFloat64("json-rpc.txfee-cap"), + EVMTimeout: v.GetDuration("json-rpc.evm-timeout"), + LogsCap: v.GetInt32("json-rpc.logs-cap"), + BlockRangeCap: v.GetInt32("json-rpc.block-range-cap"), + HTTPTimeout: v.GetDuration("json-rpc.http-timeout"), + HTTPIdleTimeout: v.GetDuration("json-rpc.http-idle-timeout"), + MaxOpenConnections: v.GetInt("json-rpc.max-open-connections"), + EnableIndexer: v.GetBool("json-rpc.enable-indexer"), + AllowIndexerGap: v.GetBool("json-rpc.allow-indexer-gap"), + MetricsAddress: v.GetString("json-rpc.metrics-address"), + FixRevertGasRefundHeight: v.GetInt64("json-rpc.fix-revert-gas-refund-height"), + ReturnDataLimit: v.GetInt64("json-rpc.return-data-limit"), + BackupGRPCBlockAddressBlockRange: backupGRPCBlockAddressBlockRange, }, TLS: TLSConfig{ CertificatePath: v.GetString("tls.certificate-path"), diff --git a/server/config/toml.go b/server/config/toml.go index b82d492c7f..e3351c7bff 100644 --- a/server/config/toml.go +++ b/server/config/toml.go @@ -106,6 +106,11 @@ fix-revert-gas-refund-height = {{ .JSONRPC.FixRevertGasRefundHeight }} # Maximum number of bytes returned from eth_call or similar invocations. return-data-limit = {{ .JSONRPC.ReturnDataLimit }} +# A list of backup grpc address with block range +# Example: "0.0.0.0:26113" = [0, 20] +backup-grpc-address-block-range = "{{ "{" }}{{ range $k, $v := .JSONRPC.BackupGRPCBlockAddressBlockRange }}" + + "\"{{ $v }}\": [{{index $k 0 }}, {{ index $k 1}}]{{ end }}{{ "}" }}" + ############################################################################### ### TLS Configuration ### ############################################################################### diff --git a/server/flags/flags.go b/server/flags/flags.go index 02dc161342..ad14f02fab 100644 --- a/server/flags/flags.go +++ b/server/flags/flags.go @@ -69,9 +69,10 @@ const ( // JSONRPCEnableMetrics enables EVM RPC metrics server. // Set to `metrics` which is hardcoded flag from go-ethereum. // https://github.com/ethereum/go-ethereum/blob/master/metrics/metrics.go#L35-L55 - JSONRPCEnableMetrics = "metrics" - JSONRPCFixRevertGasRefundHeight = "json-rpc.fix-revert-gas-refund-height" - JSONRPCReturnDataLimit = "json-rpc.return-data-limit" + JSONRPCEnableMetrics = "metrics" + JSONRPCFixRevertGasRefundHeight = "json-rpc.fix-revert-gas-refund-height" + JSONRPCReturnDataLimit = "json-rpc.return-data-limit" + JSONRPCBackupGRPCBlockAddressBlockRange = "json-rpc.backup-grpc-address-block-range" ) // EVM flags diff --git a/server/start.go b/server/start.go index 9342874eee..e40ab229a8 100644 --- a/server/start.go +++ b/server/start.go @@ -215,6 +215,7 @@ which accepts a path for the resulting pprof file. cmd.Flags().Bool(srvflags.JSONRPCEnableIndexer, false, "Enable the custom tx indexer for json-rpc") cmd.Flags().Bool(srvflags.JSONRPCAllowIndexerGap, true, "Allow block gap for the custom tx indexer for json-rpc") cmd.Flags().Bool(srvflags.JSONRPCEnableMetrics, false, "Define if EVM rpc metrics server should be enabled") + cmd.Flags().String(srvflags.JSONRPCBackupGRPCBlockAddressBlockRange, "", "Define if backup grpc and block range is available") cmd.Flags().String(srvflags.EVMTracer, config.DefaultEVMTracer, "the EVM tracer type to collect execution traces from the EVM transaction execution (json|struct|access_list|markdown)") //nolint:lll cmd.Flags().Uint64(srvflags.EVMMaxTxGasWanted, config.DefaultMaxTxGasWanted, "the gas wanted for each eth tx returned in ante handler in check tx mode") //nolint:lll diff --git a/server/util.go b/server/util.go index 4f9b24837a..be21fde89f 100644 --- a/server/util.go +++ b/server/util.go @@ -17,6 +17,7 @@ package server import ( "context" + "fmt" "net" "net/http" "os" @@ -34,6 +35,7 @@ import ( "github.com/cosmos/cosmos-sdk/server/types" "github.com/cosmos/cosmos-sdk/version" + errorsmod "cosmossdk.io/errors" tmlog "cosmossdk.io/log" cmtcmd "github.com/cometbft/cometbft/cmd/cometbft/commands" ) @@ -76,6 +78,14 @@ func AddCommands( ) } +func ParseGRPCAddress(address string) (string, error) { + _, port, err := net.SplitHostPort(address) + if err != nil { + return "", errorsmod.Wrapf(err, "invalid grpc address %s", address) + } + return fmt.Sprintf("127.0.0.1:%s", port), nil +} + func MountGRPCWebServices( router *mux.Router, grpcWeb *grpcweb.WrappedGrpcServer, diff --git a/x/evm/keeper/grpc_query.go b/x/evm/keeper/grpc_query.go index 8fb28c58cd..1b5ea03a77 100644 --- a/x/evm/keeper/grpc_query.go +++ b/x/evm/keeper/grpc_query.go @@ -31,17 +31,16 @@ import ( sdkmath "cosmossdk.io/math" sdk "github.com/cosmos/cosmos-sdk/types" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" ethparams "github.com/ethereum/go-ethereum/params" - rpctypes "github.com/evmos/ethermint/rpc/types" ethermint "github.com/evmos/ethermint/types" "github.com/evmos/ethermint/x/evm/types" + "github.com/evmos/ethermint/x/utils" ) var _ types.QueryServer = Keeper{} @@ -219,10 +218,24 @@ func (k Keeper) Code(c context.Context, req *types.QueryCodeRequest) (*types.Que } // Params implements the Query/Params gRPC method -func (k Keeper) Params(c context.Context, _ *types.QueryParamsRequest) (*types.QueryParamsResponse, error) { +func (k Keeper) Params(c context.Context, req *types.QueryParamsRequest) (*types.QueryParamsResponse, error) { ctx := sdk.UnwrapSDKContext(c) + // grpc + height, err := utils.GetHeightFromMetadata(c) + if err != nil { + return nil, err + } + // cli + if height == 0 { + height = ctx.BlockHeight() + } + for blocks, client := range k.backupQueryClients { + if int64(blocks[0]) <= height && int64(blocks[1]) >= height { + params, err := client.Params(c, req) + return params, err + } + } params := k.GetParams(ctx) - return &types.QueryParamsResponse{ Params: params, }, nil diff --git a/x/evm/keeper/keeper.go b/x/evm/keeper/keeper.go index 14fee66b9c..f1b28d11aa 100644 --- a/x/evm/keeper/keeper.go +++ b/x/evm/keeper/keeper.go @@ -71,8 +71,9 @@ type Keeper struct { hooks types.EvmHooks // Legacy subspace - ss paramstypes.Subspace - customContractFns []CustomContractFn + ss paramstypes.Subspace + customContractFns []CustomContractFn + backupQueryClients map[[2]int]types.QueryClient } // NewKeeper generates new evm module keeper @@ -87,6 +88,7 @@ func NewKeeper( tracer string, ss paramstypes.Subspace, customContractFns []CustomContractFn, + backupQueryClients map[[2]int]types.QueryClient, ) *Keeper { // ensure evm module account is set if addr := ak.GetModuleAddress(types.ModuleName); addr == nil { @@ -100,17 +102,18 @@ func NewKeeper( // NOTE: we pass in the parameter space to the CommitStateDB in order to use custom denominations for the EVM operations return &Keeper{ - cdc: cdc, - authority: authority, - accountKeeper: ak, - bankKeeper: bankKeeper, - stakingKeeper: sk, - feeMarketKeeper: fmk, - storeKey: storeKey, - objectKey: objectKey, - tracer: tracer, - ss: ss, - customContractFns: customContractFns, + cdc: cdc, + authority: authority, + accountKeeper: ak, + bankKeeper: bankKeeper, + stakingKeeper: sk, + feeMarketKeeper: fmk, + storeKey: storeKey, + objectKey: objectKey, + tracer: tracer, + ss: ss, + customContractFns: customContractFns, + backupQueryClients: backupQueryClients, } } diff --git a/x/evm/statedb/statedb_test.go b/x/evm/statedb/statedb_test.go index 5060e4abbf..7469e5738e 100644 --- a/x/evm/statedb/statedb_test.go +++ b/x/evm/statedb/statedb_test.go @@ -826,6 +826,7 @@ func newTestKeeper(t *testing.T, cms storetypes.MultiStore) (sdk.Context, *evmke "", paramstypes.Subspace{}, nil, + nil, ) ctx := sdk.NewContext(cms, tmproto.Header{}, false, log.NewNopLogger()) diff --git a/x/feemarket/keeper/grpc_query.go b/x/feemarket/keeper/grpc_query.go index 82ea42b6bd..8b58ccc233 100644 --- a/x/feemarket/keeper/grpc_query.go +++ b/x/feemarket/keeper/grpc_query.go @@ -20,17 +20,31 @@ import ( sdkmath "cosmossdk.io/math" sdk "github.com/cosmos/cosmos-sdk/types" - "github.com/evmos/ethermint/x/feemarket/types" + "github.com/evmos/ethermint/x/utils" ) var _ types.QueryServer = Keeper{} // Params implements the Query/Params gRPC method -func (k Keeper) Params(c context.Context, _ *types.QueryParamsRequest) (*types.QueryParamsResponse, error) { +func (k Keeper) Params(c context.Context, req *types.QueryParamsRequest) (*types.QueryParamsResponse, error) { ctx := sdk.UnwrapSDKContext(c) + // grpc + height, err := utils.GetHeightFromMetadata(c) + if err != nil { + return nil, err + } + // cli + if height == 0 { + height = ctx.BlockHeight() + } + for blocks, client := range k.backupQueryClients { + if int64(blocks[0]) <= height && int64(blocks[1]) >= height { + params, err := client.Params(c, req) + return params, err + } + } params := k.GetParams(ctx) - return &types.QueryParamsResponse{ Params: params, }, nil diff --git a/x/feemarket/keeper/keeper.go b/x/feemarket/keeper/keeper.go index 94730358e1..b0104a29b9 100644 --- a/x/feemarket/keeper/keeper.go +++ b/x/feemarket/keeper/keeper.go @@ -38,7 +38,8 @@ type Keeper struct { // the address capable of executing a MsgUpdateParams message. Typically, this should be the x/gov module account. authority sdk.AccAddress // Legacy subspace - ss paramstypes.Subspace + ss paramstypes.Subspace + backupQueryClients map[[2]int]types.QueryClient } // NewKeeper generates new fee market module keeper @@ -47,6 +48,7 @@ func NewKeeper( authority sdk.AccAddress, storeKey storetypes.StoreKey, ss paramstypes.Subspace, + backupQueryClients map[[2]int]types.QueryClient, ) Keeper { // ensure authority account is correctly formatted if err := sdk.VerifyAddressFormat(authority); err != nil { @@ -54,10 +56,11 @@ func NewKeeper( } return Keeper{ - cdc: cdc, - storeKey: storeKey, - authority: authority, - ss: ss, + cdc: cdc, + storeKey: storeKey, + authority: authority, + ss: ss, + backupQueryClients: backupQueryClients, } } diff --git a/x/utils/utils.go b/x/utils/utils.go new file mode 100644 index 0000000000..e496cb7230 --- /dev/null +++ b/x/utils/utils.go @@ -0,0 +1,46 @@ +// Copyright 2021 Evmos Foundation +// This file is part of Evmos' Ethermint library. +// +// The Ethermint library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The Ethermint library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the Ethermint library. If not, see https://github.com/evmos/ethermint/blob/main/LICENSE +package utils + +import ( + "context" + "strconv" + + "google.golang.org/grpc/metadata" + + errorsmod "cosmossdk.io/errors" + sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" + grpctypes "github.com/cosmos/cosmos-sdk/types/grpc" +) + +func GetHeightFromMetadata(c context.Context) (int64, error) { + if md, ok := metadata.FromIncomingContext(c); ok { + heightHeaders := md.Get(grpctypes.GRPCBlockHeightHeader) + // Get height header from the request context, if present. + if len(heightHeaders) == 1 { + height, err := strconv.ParseInt(heightHeaders[0], 10, 64) + if err != nil { + return height, errorsmod.Wrapf( + sdkerrors.ErrInvalidRequest, + "invalid height header %q: %v", grpctypes.GRPCBlockHeightHeader, err) + } + if height < 0 { + return height, errorsmod.Wrap(sdkerrors.ErrInvalidRequest, "cannot query with height < 0; please provide a valid height") + } + } + } + return 0, nil +}