Problem: backup grpc clients are not supported

app/app.go

@@ -31,10 +31,14 @@
 	"cosmossdk.io/core/appmodule"
 	runtimeservices "github.com/cosmos/cosmos-sdk/runtime/services"
 	"github.com/cosmos/cosmos-sdk/server"
+	grpc1 "github.com/cosmos/gogoproto/grpc"
 	"github.com/cosmos/gogoproto/proto"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
 
 	"github.com/gorilla/mux"
 	"github.com/spf13/cast"
+	"github.com/spf13/viper"
 
 	"cosmossdk.io/log"
 	abci "github.com/cometbft/cometbft/abci/types"
@@ -132,6 +136,7 @@
 	"github.com/evmos/ethermint/app/ante"
 	"github.com/evmos/ethermint/encoding"
 	"github.com/evmos/ethermint/ethereum/eip712"
+	srv "github.com/evmos/ethermint/server"
 	srvconfig "github.com/evmos/ethermint/server/config"
 	srvflags "github.com/evmos/ethermint/server/flags"
 	ethermint "github.com/evmos/ethermint/types"
@@ -276,6 +281,46 @@
 	configurator module.Configurator
 }
 
+func BackupQueryClients[T any](appOpts servertypes.AppOptions, interfaceRegistry types.InterfaceRegistry, queryClientFn func(grpc1.ClientConn) T) map[[2]int]T {
+	backupQueryClients := make(map[[2]int]T)
+	if v, ok := appOpts.(*viper.Viper); ok {
+		cfg, err := srvconfig.GetConfig(v)
+		if err == nil {
+			maxSendMsgSize := cfg.GRPC.MaxSendMsgSize
+			if maxSendMsgSize == 0 {
+				maxSendMsgSize = config.DefaultGRPCMaxSendMsgSize
+			}
+
+			maxRecvMsgSize := cfg.GRPC.MaxRecvMsgSize
+			if maxRecvMsgSize == 0 {
+				maxRecvMsgSize = config.DefaultGRPCMaxRecvMsgSize
+			}
+
+			for k, address := range cfg.JSONRPC.BackupGRPCBlockAddressBlockRange {
+				grpcAddr, err := srv.ParseGRPCAddress(address)
+				if err != nil {
+					continue
+				}
+
+				conn, err := grpc.Dial(
+					grpcAddr,
+					grpc.WithTransportCredentials(insecure.NewCredentials()),
+					grpc.WithDefaultCallOptions(
+						grpc.ForceCodec(codec.NewProtoCodec(interfaceRegistry).GRPCCodec()),
+						grpc.MaxCallRecvMsgSize(maxRecvMsgSize),
+						grpc.MaxCallSendMsgSize(maxSendMsgSize),
+					),
+				)
+				if err != nil {
+					continue
+				}
+				backupQueryClients[k] = queryClientFn(conn)
+			}
+		}
+	}
+	return backupQueryClients
+}
+
 // NewEthermintApp returns a reference to a new initialized Ethermint application.
 func NewEthermintApp(
 	logger log.Logger,
@@ -512,6 +557,7 @@
 		authtypes.NewModuleAddress(govtypes.ModuleName),
 		keys[feemarkettypes.StoreKey],
 		feeMarketSs,
+		BackupQueryClients(appOpts, interfaceRegistry, feemarkettypes.NewQueryClient),
 	)
 
 	// Set authority to x/gov module account to only expect the module account to update params
@@ -523,6 +569,7 @@
 		tracer,
 		evmSs,
 		nil,
+		BackupQueryClients(appOpts, interfaceRegistry, evmtypes.NewQueryClient),
 	)
 
 	// register the proposal types

server/config/config.go

@@ -16,6 +16,7 @@
 package config
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"path"
@@ -179,6 +180,8 @@
 	FixRevertGasRefundHeight int64 `mapstructure:"fix-revert-gas-refund-height"`
 	// ReturnDataLimit defines maximum number of bytes returned from `eth_call` or similar invocations
 	ReturnDataLimit int64 `mapstructure:"return-data-limit"`
+	// A list of grpc address with block range
+	BackupGRPCBlockAddressBlockRange map[[2]int]string `mapstructure:"backup-grpc-address-block-range"`
 }
 
 // TLSConfig defines the certificate and matching private key for the server.
@@ -277,26 +280,27 @@
 // DefaultJSONRPCConfig returns an EVM config with the JSON-RPC API enabled by default
 func DefaultJSONRPCConfig() *JSONRPCConfig {
 	return &JSONRPCConfig{
-		Enable:                   true,
-		API:                      GetDefaultAPINamespaces(),
-		Address:                  DefaultJSONRPCAddress,
-		WsAddress:                DefaultJSONRPCWsAddress,
-		GasCap:                   DefaultGasCap,
-		EVMTimeout:               DefaultEVMTimeout,
-		TxFeeCap:                 DefaultTxFeeCap,
-		FilterCap:                DefaultFilterCap,
-		FeeHistoryCap:            DefaultFeeHistoryCap,
-		BlockRangeCap:            DefaultBlockRangeCap,
-		LogsCap:                  DefaultLogsCap,
-		HTTPTimeout:              DefaultHTTPTimeout,
-		HTTPIdleTimeout:          DefaultHTTPIdleTimeout,
-		AllowUnprotectedTxs:      DefaultAllowUnprotectedTxs,
-		MaxOpenConnections:       DefaultMaxOpenConnections,
-		EnableIndexer:            false,
-		AllowIndexerGap:          true,
-		MetricsAddress:           DefaultJSONRPCMetricsAddress,
-		FixRevertGasRefundHeight: DefaultFixRevertGasRefundHeight,
-		ReturnDataLimit:          DefaultReturnDataLimit,
+		Enable:                           true,
+		API:                              GetDefaultAPINamespaces(),
+		Address:                          DefaultJSONRPCAddress,
+		WsAddress:                        DefaultJSONRPCWsAddress,
+		GasCap:                           DefaultGasCap,
+		EVMTimeout:                       DefaultEVMTimeout,
+		TxFeeCap:                         DefaultTxFeeCap,
+		FilterCap:                        DefaultFilterCap,
+		FeeHistoryCap:                    DefaultFeeHistoryCap,
+		BlockRangeCap:                    DefaultBlockRangeCap,
+		LogsCap:                          DefaultLogsCap,
+		HTTPTimeout:                      DefaultHTTPTimeout,
+		HTTPIdleTimeout:                  DefaultHTTPIdleTimeout,
+		AllowUnprotectedTxs:              DefaultAllowUnprotectedTxs,
+		MaxOpenConnections:               DefaultMaxOpenConnections,
+		EnableIndexer:                    false,
+		AllowIndexerGap:                  true,
+		MetricsAddress:                   DefaultJSONRPCMetricsAddress,
+		FixRevertGasRefundHeight:         DefaultFixRevertGasRefundHeight,
+		ReturnDataLimit:                  DefaultReturnDataLimit,
+		BackupGRPCBlockAddressBlockRange: make(map[[2]int]string),
 	}
 }
 
@@ -403,6 +407,18 @@
 		return Config{}, err
 	}
 
+	data := make(map[string][2]int)
+	raw := v.GetString("json-rpc.backup-grpc-address-block-range")
+	if len(raw) > 0 {
+		err = json.Unmarshal([]byte(raw), &data)
+		if err != nil {
+			return Config{}, err
+		}
+	}
+	backupGRPCBlockAddressBlockRange := make(map[[2]int]string)
+	for k, v := range data {
+		backupGRPCBlockAddressBlockRange[v] = k
+	}
 	return Config{
 		Config: cfg,
 		EVM: EVMConfig{
@@ -412,25 +428,26 @@
 			BlockSTMWorkers: v.GetInt("evm.block-stm-workers"),
 		},
 		JSONRPC: JSONRPCConfig{
-			Enable:                   v.GetBool("json-rpc.enable"),
-			API:                      v.GetStringSlice("json-rpc.api"),
-			Address:                  v.GetString("json-rpc.address"),
-			WsAddress:                v.GetString("json-rpc.ws-address"),
-			GasCap:                   v.GetUint64("json-rpc.gas-cap"),
-			FilterCap:                v.GetInt32("json-rpc.filter-cap"),
-			FeeHistoryCap:            v.GetInt32("json-rpc.feehistory-cap"),
-			TxFeeCap:                 v.GetFloat64("json-rpc.txfee-cap"),
-			EVMTimeout:               v.GetDuration("json-rpc.evm-timeout"),
-			LogsCap:                  v.GetInt32("json-rpc.logs-cap"),
-			BlockRangeCap:            v.GetInt32("json-rpc.block-range-cap"),
-			HTTPTimeout:              v.GetDuration("json-rpc.http-timeout"),
-			HTTPIdleTimeout:          v.GetDuration("json-rpc.http-idle-timeout"),
-			MaxOpenConnections:       v.GetInt("json-rpc.max-open-connections"),
-			EnableIndexer:            v.GetBool("json-rpc.enable-indexer"),
-			AllowIndexerGap:          v.GetBool("json-rpc.allow-indexer-gap"),
-			MetricsAddress:           v.GetString("json-rpc.metrics-address"),
-			FixRevertGasRefundHeight: v.GetInt64("json-rpc.fix-revert-gas-refund-height"),
-			ReturnDataLimit:          v.GetInt64("json-rpc.return-data-limit"),
+			Enable:                           v.GetBool("json-rpc.enable"),
+			API:                              v.GetStringSlice("json-rpc.api"),
+			Address:                          v.GetString("json-rpc.address"),
+			WsAddress:                        v.GetString("json-rpc.ws-address"),
+			GasCap:                           v.GetUint64("json-rpc.gas-cap"),
+			FilterCap:                        v.GetInt32("json-rpc.filter-cap"),
+			FeeHistoryCap:                    v.GetInt32("json-rpc.feehistory-cap"),
+			TxFeeCap:                         v.GetFloat64("json-rpc.txfee-cap"),
+			EVMTimeout:                       v.GetDuration("json-rpc.evm-timeout"),
+			LogsCap:                          v.GetInt32("json-rpc.logs-cap"),
+			BlockRangeCap:                    v.GetInt32("json-rpc.block-range-cap"),
+			HTTPTimeout:                      v.GetDuration("json-rpc.http-timeout"),
+			HTTPIdleTimeout:                  v.GetDuration("json-rpc.http-idle-timeout"),
+			MaxOpenConnections:               v.GetInt("json-rpc.max-open-connections"),
+			EnableIndexer:                    v.GetBool("json-rpc.enable-indexer"),
+			AllowIndexerGap:                  v.GetBool("json-rpc.allow-indexer-gap"),
+			MetricsAddress:                   v.GetString("json-rpc.metrics-address"),
+			FixRevertGasRefundHeight:         v.GetInt64("json-rpc.fix-revert-gas-refund-height"),
+			ReturnDataLimit:                  v.GetInt64("json-rpc.return-data-limit"),
+			BackupGRPCBlockAddressBlockRange: backupGRPCBlockAddressBlockRange,
 		},
 		TLS: TLSConfig{
 			CertificatePath: v.GetString("tls.certificate-path"),

server/config/toml.go

@@ -106,6 +106,10 @@
 # Maximum number of bytes returned from eth_call or similar invocations.
 return-data-limit = {{ .JSONRPC.ReturnDataLimit }}
 
+# A list of backup grpc address with block range
+# Example: "0.0.0.0:26113" = [0, 20]
+backup-grpc-address-block-range = "{{ "{" }}{{ range $k, $v := .JSONRPC.BackupGRPCBlockAddressBlockRange }}\"{{ $v }}\": [{{index $k 0 }}, {{ index $k 1}}]{{ end }}{{ "}" }}"
+
 ###############################################################################
 ###                             TLS Configuration                           ###
 ###############################################################################

server/flags/flags.go

@@ -69,9 +69,10 @@ const (
 	// JSONRPCEnableMetrics enables EVM RPC metrics server.
 	// Set to `metrics` which is hardcoded flag from go-ethereum.
 	// https://github.com/ethereum/go-ethereum/blob/master/metrics/metrics.go#L35-L55
-	JSONRPCEnableMetrics            = "metrics"
-	JSONRPCFixRevertGasRefundHeight = "json-rpc.fix-revert-gas-refund-height"
-	JSONRPCReturnDataLimit          = "json-rpc.return-data-limit"
+	JSONRPCEnableMetrics                    = "metrics"
+	JSONRPCFixRevertGasRefundHeight         = "json-rpc.fix-revert-gas-refund-height"
+	JSONRPCReturnDataLimit                  = "json-rpc.return-data-limit"
+	JSONRPCBackupGRPCBlockAddressBlockRange = "json-rpc.backup-grpc-address-block-range"
 )
 
 // EVM flags

server/start.go

@@ -215,6 +215,7 @@ which accepts a path for the resulting pprof file.
 	cmd.Flags().Bool(srvflags.JSONRPCEnableIndexer, false, "Enable the custom tx indexer for json-rpc")
 	cmd.Flags().Bool(srvflags.JSONRPCAllowIndexerGap, true, "Allow block gap for the custom tx indexer for json-rpc")
 	cmd.Flags().Bool(srvflags.JSONRPCEnableMetrics, false, "Define if EVM rpc metrics server should be enabled")
+	cmd.Flags().String(srvflags.JSONRPCBackupGRPCBlockAddressBlockRange, "", "Define if backup grpc and block range is available")
 
 	cmd.Flags().String(srvflags.EVMTracer, config.DefaultEVMTracer, "the EVM tracer type to collect execution traces from the EVM transaction execution (json|struct|access_list|markdown)") //nolint:lll
 	cmd.Flags().Uint64(srvflags.EVMMaxTxGasWanted, config.DefaultMaxTxGasWanted, "the gas wanted for each eth tx returned in ante handler in check tx mode")                                 //nolint:lll

server/util.go

@@ -17,6 +17,7 @@ package server
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"net/http"
 	"os"
@@ -34,6 +35,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/server/types"
 	"github.com/cosmos/cosmos-sdk/version"
 
+	errorsmod "cosmossdk.io/errors"
 	tmlog "cosmossdk.io/log"
 	cmtcmd "github.com/cometbft/cometbft/cmd/cometbft/commands"
 )
@@ -76,6 +78,14 @@ func AddCommands(
 	)
 }
 
+func ParseGRPCAddress(address string) (string, error) {
+	_, port, err := net.SplitHostPort(address)
+	if err != nil {
+		return "", errorsmod.Wrapf(err, "invalid grpc address %s", address)
+	}
+	return fmt.Sprintf("127.0.0.1:%s", port), nil
+}
+
 func MountGRPCWebServices(
 	router *mux.Router,
 	grpcWeb *grpcweb.WrappedGrpcServer,

x/evm/keeper/grpc_query.go

@@ -31,17 +31,16 @@
 
 	sdkmath "cosmossdk.io/math"
 	sdk "github.com/cosmos/cosmos-sdk/types"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core"
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	ethparams "github.com/ethereum/go-ethereum/params"
-
 	rpctypes "github.com/evmos/ethermint/rpc/types"
 	ethermint "github.com/evmos/ethermint/types"
 	"github.com/evmos/ethermint/x/evm/types"
+	"github.com/evmos/ethermint/x/utils"
 )
 
 var _ types.QueryServer = Keeper{}
@@ -219,10 +218,24 @@
 }
 
 // Params implements the Query/Params gRPC method
-func (k Keeper) Params(c context.Context, _ *types.QueryParamsRequest) (*types.QueryParamsResponse, error) {
+func (k Keeper) Params(c context.Context, req *types.QueryParamsRequest) (*types.QueryParamsResponse, error) {
 	ctx := sdk.UnwrapSDKContext(c)
+	// grpc
+	height, err := utils.GetHeightFromMetadata(c)
+	if err != nil {
+		return nil, err
+	}
+	// cli
+	if height == 0 {
+		height = ctx.BlockHeight()
+	}
+	for blocks, client := range k.backupQueryClients {
+		if int64(blocks[0]) <= height && int64(blocks[1]) >= height {
+			params, err := client.Params(c, req)
+			return params, err
+		}
+	}
 	params := k.GetParams(ctx)
-
 	return &types.QueryParamsResponse{
 		Params: params,
 	}, nil

x/evm/keeper/keeper.go

@@ -71,8 +71,9 @@ type Keeper struct {
 	hooks types.EvmHooks
 
 	// Legacy subspace
-	ss                paramstypes.Subspace
-	customContractFns []CustomContractFn
+	ss                 paramstypes.Subspace
+	customContractFns  []CustomContractFn
+	backupQueryClients map[[2]int]types.QueryClient
 }
 
 // NewKeeper generates new evm module keeper
@@ -87,6 +88,7 @@ func NewKeeper(
 	tracer string,
 	ss paramstypes.Subspace,
 	customContractFns []CustomContractFn,
+	backupQueryClients map[[2]int]types.QueryClient,
 ) *Keeper {
 	// ensure evm module account is set
 	if addr := ak.GetModuleAddress(types.ModuleName); addr == nil {
@@ -100,17 +102,18 @@ func NewKeeper(
 
 	// NOTE: we pass in the parameter space to the CommitStateDB in order to use custom denominations for the EVM operations
 	return &Keeper{
-		cdc:               cdc,
-		authority:         authority,
-		accountKeeper:     ak,
-		bankKeeper:        bankKeeper,
-		stakingKeeper:     sk,
-		feeMarketKeeper:   fmk,
-		storeKey:          storeKey,
-		objectKey:         objectKey,
-		tracer:            tracer,
-		ss:                ss,
-		customContractFns: customContractFns,
+		cdc:                cdc,
+		authority:          authority,
+		accountKeeper:      ak,
+		bankKeeper:         bankKeeper,
+		stakingKeeper:      sk,
+		feeMarketKeeper:    fmk,
+		storeKey:           storeKey,
+		objectKey:          objectKey,
+		tracer:             tracer,
+		ss:                 ss,
+		customContractFns:  customContractFns,
+		backupQueryClients: backupQueryClients,
 	}
 }