diff --git a/.env b/.env index 78da0a0..d7b38f5 100644 --- a/.env +++ b/.env @@ -27,7 +27,7 @@ KAVA_PRUNING_HOST_EVM_RPC_PORT=8555 KAVA_PRUNING_HOST_COSMOS_RPC_PORT=26667 PROXY_CONTAINER_PORT=7777 -PROXY_CONTAINER_EVM_RPC_DATA_PORT=7778 +PROXY_CONTAINER_EVM_RPC_PRUNING_PORT=7778 PROXY_HOST_PORT=7777 PROXY_CONTAINER_DEBUG_PORT=2345 PROXY_HOST_DEBUG_PORT=2345 @@ -35,10 +35,12 @@ PROXY_HOST_DEBUG_PORT=2345 ##### E2E Testing Config TEST_PROXY_SERVICE_EVM_RPC_URL=http://localhost:7777 TEST_PROXY_SERVICE_EVM_RPC_HOSTNAME=localhost:7777 -TEST_PROXY_SERVICE_EVM_RPC_DATA_URL=http://localhost:7778 +TEST_PROXY_SERVICE_EVM_RPC_PRUNING_URL=http://localhost:7778 TEST_PROXY_BACKEND_EVM_RPC_HOST_URL=http://localhost:8545 TEST_DATABASE_ENDPOINT_URL=localhost:5432 TEST_PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-validator:8545,localhost:7778>http://kava-pruning:8545 +TEST_PROXY_HEIGHT_BASED_ROUTING_ENABLED=true +TEST_PROXY_PRUNING_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-pruning:8545,localhost:7778>http://kava-pruning:8545 # What level of logging to use for service objects constructed during # unit tests TEST_SERVICE_LOG_LEVEL=ERROR @@ -57,6 +59,11 @@ HTTP_READ_TIMEOUT_SECONDS=30 HTTP_WRITE_TIMEOUT_SECONDS=60 # Address of the origin server to proxy all requests to PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-validator:8545,localhost:7778>http://kava-pruning:8545 +# height-based routing will look at the height of an incoming EVM request +# iff. the height is "latest", it routes to the corresponding PROXY_PRUNING_BACKEND_HOST_URL_MAP value +# otherwise, it falls back to the value in PROXY_BACKEND_HOST_URL_MAP +PROXY_HEIGHT_BASED_ROUTING_ENABLED=true +PROXY_PRUNING_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-pruning:8545,localhost:7778>http://kava-pruning:8545 # Configuration for the servcie to connect to it's database DATABASE_NAME=postgres DATABASE_ENDPOINT_URL=postgres:5432 diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index ea08e5d..6a610f7 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -98,6 +98,17 @@ make ready e2e-test For details on the local E2E setup, see [the `docker` directory](./docker/README.md). +#### Against testnet + +The Continuous Integration (CI) for this repo is setup to run a local proxy service with database & redis, but configures the service to use public testnet urls for the kava requests. This allows for testing the proxy service against a production-like environment (requests are routed to public testnet). + +You can emulate the CI configuration in your local environment: +```bash +make ci-setup +``` + +At that point, running `make e2e-test` will run the end-to-end tests with requests routing to public testnet. + ## Test Coverage Report The test commands `make test`, `make unit-test`, and `make e2e-test` generate a `cover.out` raw test coverage report. The coverage can be converted into a user-friendly webpage: diff --git a/Makefile b/Makefile index 34887f7..7d05207 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,14 @@ unit-test: e2e-test: go test -count=1 -v -cover -coverprofile cover.out --race ./... -run "^TestE2ETest*" +.PHONY: ci-setup +# set up your local environment such that running `make e2e-test` runs against testnet (like in CI) +ci-setup: + docker compose -f ci.docker-compose.yml pull + docker compose -f ci.docker-compose.yml up -d --build --force-recreate + PROXY_CONTAINER_PORT=7777 bash scripts/wait-for-proxy-service-running.sh + PROXY_CONTAINER_PORT=7777 MINIMUM_REQUIRED_PARTITIONS=30 bash scripts/wait-for-proxy-service-running.sh + .PHONY: it # run any test matching the provided pattern, can pass a regex or a string # of the exact test to run diff --git a/architecture/MIDDLEWARE.MD b/architecture/MIDDLEWARE.MD index 0685c71..034d1cf 100644 --- a/architecture/MIDDLEWARE.MD +++ b/architecture/MIDDLEWARE.MD @@ -61,27 +61,7 @@ server := &http.Server{ 1. Times the roundtrip latency for the response from the backend origin server and stores the latency in the context key `X-KAVA-PROXY-ORIGIN-ROUNDTRIP-LATENCY-MILLISECONDS` for use by other middleware. -#### Configuration - -The proxy is configured to route requests based on their incoming Host. These are controlled via the -`PROXY_BACKEND_HOST_URL_MAP` environment variable. - -As an example, consider a value of `localhost:7777>http://kava:8545,localhost:7778>http://kava:8545`. -This value is parsed into a map that looks like the following: -``` -{ - "localhost:7777" => "http://kava:8545", - "localhost:7778" => "http://kava:8545", -} -``` -Any request to the service will be routed according to this map. -ie. all requests to local ports 7777 & 7778 get forwarded to `http://kava:8545` - -Implementations of the [`Proxies` interface](../service/proxy.go#L13) contain logic for deciding -the backend host url to which a request is routed. This is used in the ProxyRequestMiddleware to -route requests. - -Any request made to a host not in the map responds 502 Bad Gateway. +See [Proxy Routing](./PROXY_ROUTING.md) for details on configuration and how requests are routed. ### After Proxy Middleware diff --git a/architecture/PROXY_ROUTING.md b/architecture/PROXY_ROUTING.md new file mode 100644 index 0000000..0a730d1 --- /dev/null +++ b/architecture/PROXY_ROUTING.md @@ -0,0 +1,152 @@ +# Proxy Routing + +The proxy chooses where to route a request primarily by the incoming Host URL to which the client +originally made their request. The routing is configured by maps of the Host to a backend url in +the environment variables. + +All possible configurations use the `PROXY_BACKEND_HOST_URL_MAP` environment variable. This encodes +the default backend to route all requests from a given host. Additional functionality is available +via the `PROXY_HEIGHT_BASED_ROUTING_ENABLED` env variable (see [Rudimentary Sharding](#rudimentary-sharding)). + +Consider the simplest case: a host-based-only routing proxy configured for one host +``` +PROXY_HEIGHT_BASED_ROUTING_ENABLED=false +PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava:8545 +``` +This value is parsed into a map that looks like the following: +``` +{ + "localhost:7777" => "http://kava:8545", +} +``` +Any request to the service will be routed according to this map. +ie. all requests to local port 7777 get forwarded to `http://kava:8545` + +Implementations of the [`Proxies` interface](../service/proxy.go#L13) contain logic for deciding +the backend host url to which a request is routed. This is used in the ProxyRequestMiddleware to +route requests. + +Any request made to a host not in the map responds 502 Bad Gateway. + +## More Examples of Host-only routing + +Here is a diagram of the above network topology: + +![Proxy Service configured for one host](images/proxy_service_simple_one_host.jpg) + +In this simple configuration of only having default hosts, you can do many things: + +**Many hosts -> One backend** + +``` +PROXY_HEIGHT_BASED_ROUTING_ENABLED=false +PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava:8545,localhost:7778>http://kava:8545 +``` +This value is parsed into a map that looks like the following: +``` +{ + "localhost:7777" => "http://kava:8545", + "localhost:7778" => "http://kava:8545", +} +``` +All requests to local ports 7777 & 7778 route to the same cluster at kava:8545 + +![Proxy Service configured for many hosts for one backend](images/proxy_service_many_hosts_one_backend.jpg) + +**Many hosts -> Many backends** + +``` +PROXY_HEIGHT_BASED_ROUTING_ENABLED=false +PROXY_BACKEND_HOST_URL_MAP=evm.kava.io>http://kava-pruning:8545,evm.data.kava.io>http://kava-archive:8545 +``` +This value is parsed into a map that looks like the following: +``` +{ + "evm.kava.io" => "http://kava-pruning:8545", + "evm.data.kava.io" => "http://kava-archive:8545", +} +``` +Requests made to evm.kava.io route to a pruning cluster. +Those made to evm.data.kava.io route to an archive cluster. + +![Proxy Service configured for many hosts with many backends](images/proxy_service_many_hosts_many_backends.jpg) + +## Rudimentary Sharding + +Now suppose you want multiple backends for the same host. + +The proxy service supports height-based routing to direct requests that only require the most recent +block to a different cluster. +This support is handled via the [`HeightShardingProxies` implementation](../service/shard.go#L16). + +This is configured via the `PROXY_HEIGHT_BASED_ROUTING_ENABLED` and `PROXY_PRUNING_BACKEND_HOST_URL_MAP` +environment variables. +* `PROXY_HEIGHT_BASED_ROUTING_ENABLED` - flag to toggle this functionality +* `PROXY_PRUNING_BACKEND_HOST_URL_MAP` - like `PROXY_BACKEND_HOST_URL_MAP`, but only used for JSON-RPC + requests that target the latest block (or are stateless, like `eth_chainId`, `eth_coinbase`, etc). + +For example, to lighten the load for your resource-intensive (& expensive) archive cluster, you can +route all requests for the "latest" block to a less resource-intensive (& cheaper) pruning cluster: +``` +PROXY_HEIGHT_BASED_ROUTING_ENABLED=true +PROXY_BACKEND_HOST_URL_MAP=evm.data.kava.io>http://kava-archive:8545 +PROXY_PRUNING_BACKEND_HOST_URL_MAP=evm.data.kava.io>http://kava-pruning:8545 +``` +This value is parsed into a map that looks like the following: +``` +{ + "default": { + "evm.data.kava.io" => "http://kava-archive:8545", + }, + "pruning": { + "evm.data.kava.io" => "http://kava-pruning:8545", + } +} +``` +All traffic to evm.data.kava.io that targets the latest block (or requires no history) routes to the pruning cluster. +Otherwise, all traffic is sent to the archive cluster. + +![Proxy Service configured with rudimentary sharding](images/proxy_service_rudimentary_sharding.jpg) + +### Default vs Pruning Backend Routing + +When `PROXY_HEIGHT_BASED_ROUTING_ENABLED` is `true`, the following cases will cause requests to route +to the the backend url defined in `PROXY_PRUNING_BACKEND_HOST_URL_MAP` (if present): +* requests that include any of the following block tags: + * `"latest"` + * `"finalized"` + * `"pending"` + * `"safe"` + * empty/missing block tag (interpreted as `"latest"`) +* requests for methods that require no historic state, including transaction broadcasting + * for a full list of methods, see [`NoHistoryMethods`](../decode/evm_rpc.go#L89) + +All other requests fallback to the default backend url defined in `PROXY_BACKEND_HOST_URL_MAP`. +This includes +* requests for hosts not included in `PROXY_PRUNING_BACKEND_HOST_URL_MAP` +* requests targeting any specific height by number + * NOTE: the service does not track the current height of the chain. if the tip of the chain is at + block 1000, a query for block 1000 will still route to the default (not pruning) backend +* requests for methods that use block hash, like `eth_getBlockByHash` +* requests with unparsable (invalid) block numbers +* requests for block tag `"earliest"` + +The service will panic on startup if a host in `PROXY_PRUNING_BACKEND_HOST_URL_MAP` is not present +in `PROXY_BACKEND_HOST_URL_MAP`. + +Any request made to a host not in the `PROXY_BACKEND_HOST_URL_MAP` map responds 502 Bad Gateway. + +## Metrics + +When metrics are enabled, the `proxied_request_metrics` table tracks the backend to which requests +are routed in the `response_backend` column. + +When height-based sharding is disabled (`PROXY_HEIGHT_BASED_ROUTING_ENABLED=false`), the value is +always `DEFAULT`. + +When enabled, the column will have one of the following values: +* `DEFAULT` - the request was routed to the backend defined in `PROXY_BACKEND_HOST_URL_MAP` +* `PRUNING` - the request was routed to the backend defined in `PROXY_PRUNING_BACKEND_HOST_URL_MAP` + +Additionally, the actual URL to which the request is routed to is tracked in the +`response_backend_route` column. diff --git a/architecture/images/proxy_service_many_hosts_many_backends.jpg b/architecture/images/proxy_service_many_hosts_many_backends.jpg new file mode 100644 index 0000000..1ea1274 Binary files /dev/null and b/architecture/images/proxy_service_many_hosts_many_backends.jpg differ diff --git a/architecture/images/proxy_service_many_hosts_one_backend.jpg b/architecture/images/proxy_service_many_hosts_one_backend.jpg new file mode 100644 index 0000000..772e316 Binary files /dev/null and b/architecture/images/proxy_service_many_hosts_one_backend.jpg differ diff --git a/architecture/images/proxy_service_rudimentary_sharding.jpg b/architecture/images/proxy_service_rudimentary_sharding.jpg new file mode 100644 index 0000000..7f6d009 Binary files /dev/null and b/architecture/images/proxy_service_rudimentary_sharding.jpg differ diff --git a/architecture/images/proxy_service_simple_one_host.jpg b/architecture/images/proxy_service_simple_one_host.jpg new file mode 100644 index 0000000..8afabb9 Binary files /dev/null and b/architecture/images/proxy_service_simple_one_host.jpg differ diff --git a/ci.docker-compose.yml b/ci.docker-compose.yml index d0bd405..5d3be4f 100644 --- a/ci.docker-compose.yml +++ b/ci.docker-compose.yml @@ -22,12 +22,14 @@ services: dockerfile: ci.Dockerfile env_file: .env environment: + PROXY_HEIGHT_BASED_ROUTING_ENABLED: true # use public testnet as backend origin server to avoid having # to self-host a beefy Github Action runner # to build and run a kava node each execution - PROXY_BACKEND_HOST_URL_MAP: localhost:7777>https://evmrpc.internal.testnet.proxy.kava.io,localhost:7778>https://evmrpcdata.internal.testnet.proxy.kava.io + PROXY_BACKEND_HOST_URL_MAP: localhost:7777>https://evmrpcdata.internal.testnet.proxy.kava.io,localhost:7778>https://evmrpc.internal.testnet.proxy.kava.io + PROXY_PRUNING_BACKEND_HOST_URL_MAP: localhost:7777>https://evmrpc.internal.testnet.proxy.kava.io EVM_QUERY_SERVICE_URL: https://evmrpc.internal.testnet.proxy.kava.io ports: - "${PROXY_HOST_PORT}:${PROXY_CONTAINER_PORT}" - - "${PROXY_CONTAINER_EVM_RPC_DATA_PORT}:${PROXY_CONTAINER_PORT}" + - "${PROXY_CONTAINER_EVM_RPC_PRUNING_PORT}:${PROXY_CONTAINER_PORT}" - "${PROXY_HOST_DEBUG_PORT}:${PROXY_CONTAINER_DEBUG_PORT}" diff --git a/clients/database/migrations/20231013122742_add_response_backend.up.sql b/clients/database/migrations/20231013122742_add_response_backend.up.sql new file mode 100644 index 0000000..834b35d --- /dev/null +++ b/clients/database/migrations/20231013122742_add_response_backend.up.sql @@ -0,0 +1,9 @@ +-- add response backend column, backfilling with "DEFAULT" (the only value to exist up until now) +-- new metrics that omit its value are assumed to have been routed to "DEFAULT" backend +-- also add response backend route. this is the backend url the request was routed to. +ALTER TABLE + IF EXISTS proxied_request_metrics +ADD + response_backend character varying DEFAULT 'DEFAULT', +ADD + response_backend_route character varying; diff --git a/clients/database/request_metric.go b/clients/database/request_metric.go index d921f88..59c7d61 100644 --- a/clients/database/request_metric.go +++ b/clients/database/request_metric.go @@ -27,6 +27,8 @@ type ProxiedRequestMetric struct { UserAgent *string Referer *string Origin *string + ResponseBackend string + ResponseBackendRoute string } // Save saves the current ProxiedRequestMetric to diff --git a/config/config.go b/config/config.go index 06e2d73..25a6f80 100644 --- a/config/config.go +++ b/config/config.go @@ -17,6 +17,9 @@ type Config struct { LogLevel string ProxyBackendHostURLMapRaw string ProxyBackendHostURLMapParsed map[string]url.URL + EnableHeightBasedRouting bool + ProxyPruningBackendHostURLMapRaw string + ProxyPruningBackendHostURLMap map[string]url.URL EvmQueryServiceURL string DatabaseName string DatabaseEndpointURL string @@ -43,6 +46,8 @@ const ( LOG_LEVEL_ENVIRONMENT_KEY = "LOG_LEVEL" DEFAULT_LOG_LEVEL = "INFO" PROXY_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY = "PROXY_BACKEND_HOST_URL_MAP" + PROXY_HEIGHT_BASED_ROUTING_ENABLED_KEY = "PROXY_HEIGHT_BASED_ROUTING_ENABLED" + PROXY_PRUNING_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY = "PROXY_PRUNING_BACKEND_HOST_URL_MAP" PROXY_SERVICE_PORT_ENVIRONMENT_KEY = "PROXY_SERVICE_PORT" DATABASE_NAME_ENVIRONMENT_KEY = "DATABASE_NAME" DATABASE_ENDPOINT_URL_ENVIRONMENT_KEY = "DATABASE_ENDPOINT_URL" @@ -81,6 +86,8 @@ const ( DEFAULT_DATABASE_WRITE_TIMEOUT_SECONDS = 10 ) +var ErrEmptyHostMap = errors.New("backend host url map is empty") + // EnvOrDefault fetches an environment variable value, or if not set returns the fallback value func EnvOrDefault(key string, fallback string) string { if val, ok := os.LookupEnv(key); ok { @@ -141,8 +148,9 @@ func ParseRawProxyBackendHostURLMap(raw string) (map[string]url.URL, error) { entries := strings.Split(raw, PROXY_BACKEND_HOST_URL_MAP_ENTRY_DELIMITER) - if len(entries) < 1 { - return hostURLMap, fmt.Errorf("found zero mappings delimited by %s in %s", PROXY_BACKEND_HOST_URL_MAP_ENTRY_DELIMITER, raw) + if raw == "" || len(entries) < 1 { + extraErr := fmt.Errorf("found zero mappings delimited by %s in %s", PROXY_BACKEND_HOST_URL_MAP_ENTRY_DELIMITER, raw) + return hostURLMap, errors.Join(ErrEmptyHostMap, extraErr) } for _, entry := range entries { @@ -175,15 +183,20 @@ func ParseRawProxyBackendHostURLMap(raw string) (map[string]url.URL, error) { // function of the Config package before use func ReadConfig() Config { rawProxyBackendHostURLMap := os.Getenv(PROXY_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY) - // best effort to pares, callers are responsible for validating + rawProxyPruningBackendHostURLMap := os.Getenv(PROXY_PRUNING_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY) + // best effort to parse, callers are responsible for validating // before using any values read parsedProxyBackendHostURLMap, _ := ParseRawProxyBackendHostURLMap(rawProxyBackendHostURLMap) + parsedProxyPruningBackendHostURLMap, _ := ParseRawProxyBackendHostURLMap(rawProxyPruningBackendHostURLMap) return Config{ ProxyServicePort: os.Getenv(PROXY_SERVICE_PORT_ENVIRONMENT_KEY), LogLevel: EnvOrDefault(LOG_LEVEL_ENVIRONMENT_KEY, DEFAULT_LOG_LEVEL), ProxyBackendHostURLMapRaw: rawProxyBackendHostURLMap, ProxyBackendHostURLMapParsed: parsedProxyBackendHostURLMap, + EnableHeightBasedRouting: EnvOrDefaultBool(PROXY_HEIGHT_BASED_ROUTING_ENABLED_KEY, false), + ProxyPruningBackendHostURLMapRaw: rawProxyPruningBackendHostURLMap, + ProxyPruningBackendHostURLMap: parsedProxyPruningBackendHostURLMap, DatabaseName: os.Getenv(DATABASE_NAME_ENVIRONMENT_KEY), DatabaseEndpointURL: os.Getenv(DATABASE_ENDPOINT_URL_ENVIRONMENT_KEY), DatabaseUserName: os.Getenv(DATABASE_USERNAME_ENVIRONMENT_KEY), diff --git a/config/config_test.go b/config/config_test.go index 8c02708..c29375a 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -9,9 +9,11 @@ import ( ) var ( - proxyServicePort = "7777" - randomEnvironmentVariableKey = "TEST_KAVA_RANDOM_VALUE" - proxyServiceBackendHostURLMap = os.Getenv("TEST_PROXY_BACKEND_HOST_URL_MAP") + proxyServicePort = "7777" + randomEnvironmentVariableKey = "TEST_KAVA_RANDOM_VALUE" + proxyServiceBackendHostURLMap = os.Getenv("TEST_PROXY_BACKEND_HOST_URL_MAP") + proxyServiceHeightBasedRouting = os.Getenv("TEST_PROXY_HEIGHT_BASED_ROUTING_ENABLED") + proxyServicePruningBackendHostURLMap = os.Getenv("TEST_PROXY_PRUNING_BACKEND_HOST_URL_MAP") ) func TestUnitTestEnvODefaultReturnsDefaultIfEnvironmentVariableNotSet(t *testing.T) { @@ -46,8 +48,15 @@ func TestUnitTestReadConfigReturnsConfigWithValuesFromEnv(t *testing.T) { assert.Equal(t, proxyServicePort, readConfig.ProxyServicePort) } +func TestUnitTestParseHostMapReturnsErrEmptyHostMapWhenEmpty(t *testing.T) { + _, err := config.ParseRawProxyBackendHostURLMap("") + assert.ErrorIs(t, err, config.ErrEmptyHostMap) +} + func setDefaultEnv() { os.Setenv(config.PROXY_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY, proxyServiceBackendHostURLMap) + os.Setenv(config.PROXY_HEIGHT_BASED_ROUTING_ENABLED_KEY, proxyServiceHeightBasedRouting) + os.Setenv(config.PROXY_PRUNING_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY, proxyServicePruningBackendHostURLMap) os.Setenv(config.PROXY_SERVICE_PORT_ENVIRONMENT_KEY, proxyServicePort) os.Setenv(config.LOG_LEVEL_ENVIRONMENT_KEY, config.DEFAULT_LOG_LEVEL) } diff --git a/config/validate.go b/config/validate.go index 446fb1c..901fb37 100644 --- a/config/validate.go +++ b/config/validate.go @@ -3,6 +3,7 @@ package config import ( "errors" "fmt" + "net/url" "strconv" ) @@ -20,6 +21,7 @@ var ( func Validate(config Config) error { var validLogLevel bool var allErrs error + var err error for _, validLevel := range ValidLogLevels { if config.LogLevel == validLevel { @@ -32,10 +34,20 @@ func Validate(config Config) error { allErrs = fmt.Errorf("invalid %s specified %s, supported values are %v", LOG_LEVEL_ENVIRONMENT_KEY, config.LogLevel, ValidLogLevels) } - _, err := ParseRawProxyBackendHostURLMap(config.ProxyBackendHostURLMapRaw) + if err = validateHostURLMap(config.ProxyBackendHostURLMapRaw, false); err != nil { + allErrs = errors.Join(allErrs, fmt.Errorf("invalid %s specified %s", PROXY_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY, config.ProxyBackendHostURLMapRaw), err) + } - if err != nil { - allErrs = errors.Join(allErrs, fmt.Errorf("invalid %s specified %s", PROXY_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY, config.ProxyBackendHostURLMapRaw)) + if err = validateHostURLMap(config.ProxyPruningBackendHostURLMapRaw, true); err != nil { + allErrs = errors.Join(allErrs, fmt.Errorf("invalid %s specified %s", PROXY_PRUNING_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY, config.ProxyPruningBackendHostURLMapRaw), err) + } + + if err = validateDefaultHostMapContainsHosts( + PROXY_PRUNING_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY, + config.ProxyBackendHostURLMapParsed, + config.ProxyPruningBackendHostURLMap, + ); err != nil { + allErrs = errors.Join(allErrs, err) } _, err = strconv.Atoi(config.ProxyServicePort) @@ -50,3 +62,24 @@ func Validate(config Config) error { return allErrs } + +// validateHostURLMap validates a raw backend host URL map, optionally allowing the map to be empty +func validateHostURLMap(raw string, allowEmpty bool) error { + _, err := ParseRawProxyBackendHostURLMap(raw) + if allowEmpty && errors.Is(err, ErrEmptyHostMap) { + err = nil + } + return err +} + +// validateDefaultHostMapContainsHosts returns an error if there are hosts in hostMap that +// are not in defaultHostMap +// example: hosts in the pruning map should always have a default fallback backend +func validateDefaultHostMapContainsHosts(mapName string, defaultHostsMap, hostsMap map[string]url.URL) error { + for host := range hostsMap { + if _, found := defaultHostsMap[host]; !found { + return fmt.Errorf("host %s is in %s but not in default host map", host, mapName) + } + } + return nil +} diff --git a/config/validate_test.go b/config/validate_test.go index da054d2..9a70f63 100644 --- a/config/validate_test.go +++ b/config/validate_test.go @@ -45,6 +45,26 @@ func TestUnitTestValidateConfigReturnsErrorIfInvalidProxyBackendHostURL(t *testi assert.NotNil(t, err) } +func TestUnitTestValidateConfigReturnsNoErrorWhenPruningProxyBackendHostURLIsEmpty(t *testing.T) { + testConfig := defaultConfig + testConfig.ProxyPruningBackendHostURLMapRaw = "" + + err := config.Validate(testConfig) + + assert.Nil(t, err) +} + +func TestUnitTestValidateConfigReturnsErrorWhenPruningMapHasHostsNotInDefault(t *testing.T) { + // pruning map cannot contain hosts that aren't in default map + testConfig := defaultConfig + testConfig.ProxyPruningBackendHostURLMapRaw = "not-in-default:1234>http://mysterybackend:42" + testConfig.ProxyPruningBackendHostURLMap, _ = config.ParseRawProxyBackendHostURLMap(testConfig.ProxyPruningBackendHostURLMapRaw) + + err := config.Validate(testConfig) + + assert.NotNil(t, err) +} + func TestUnitTestValidateConfigReturnsErrorIfInvalidProxyBackendHostURLComponents(t *testing.T) { testConfig := defaultConfig testConfig.ProxyBackendHostURLMapRaw = "localhost:7777,localhost:7778>http://kava:8545$^,localhost:7777>http://kava:8545" @@ -54,6 +74,15 @@ func TestUnitTestValidateConfigReturnsErrorIfInvalidProxyBackendHostURLComponent assert.NotNil(t, err) } +func TestUnitTestValidateConfigReturnsErrorIfInvalidProxyPruningBackendHostURLComponents(t *testing.T) { + testConfig := defaultConfig + testConfig.ProxyPruningBackendHostURLMapRaw = "localhost:7777,localhost:7778>http://kava:8545$^,localhost:7777>http://kava:8545" + + err := config.Validate(testConfig) + + assert.NotNil(t, err) +} + func TestUnitTestValidateConfigReturnsErrorIfInvalidProxyServicePort(t *testing.T) { testConfig := defaultConfig testConfig.ProxyServicePort = "abc" diff --git a/decode/evm_rpc.go b/decode/evm_rpc.go index 5d0981c..0b54b05 100644 --- a/decode/evm_rpc.go +++ b/decode/evm_rpc.go @@ -100,6 +100,7 @@ var NoHistoryMethods = []string{ "eth_hashrate", "eth_gasPrice", "eth_accounts", + "eth_blockNumber", "eth_sign", "eth_signTransaction", "eth_sendTransaction", @@ -248,13 +249,13 @@ func lookupBlockNumberFromHashParam(ctx context.Context, evmClient *ethclient.Cl return 0, fmt.Errorf(fmt.Sprintf("error decoding block hash param from params %+v at index %d", params, paramIndex)) } - block, err := evmClient.BlockByHash(ctx, common.HexToHash(blockHash)) + header, err := evmClient.HeaderByHash(ctx, common.HexToHash(blockHash)) if err != nil { return 0, err } - return block.Number().Int64(), nil + return header.Number.Int64(), nil } // Generic method to parse the block number from a set of params diff --git a/docker-compose.yml b/docker-compose.yml index 2625654..fc227e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,7 +58,7 @@ services: env_file: .env ports: - "${PROXY_HOST_PORT}:${PROXY_CONTAINER_PORT}" - - "${PROXY_CONTAINER_EVM_RPC_DATA_PORT}:${PROXY_CONTAINER_PORT}" + - "${PROXY_CONTAINER_EVM_RPC_PRUNING_PORT}:${PROXY_CONTAINER_PORT}" - "${PROXY_HOST_DEBUG_PORT}:${PROXY_CONTAINER_DEBUG_PORT}" cap_add: - SYS_PTRACE # Allows for attaching debugger to process in this container diff --git a/main_test.go b/main_test.go index 851bbd5..560db62 100644 --- a/main_test.go +++ b/main_test.go @@ -2,18 +2,24 @@ package main_test import ( "context" + "fmt" "os" + "strconv" "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rpc" + "github.com/kava-labs/kava-proxy-service/clients/database" "github.com/kava-labs/kava-proxy-service/decode" "github.com/kava-labs/kava-proxy-service/logging" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + "github.com/kava-labs/kava-proxy-service/service" ) const ( @@ -32,9 +38,11 @@ var ( return logger }() - proxyServiceURL = os.Getenv("TEST_PROXY_SERVICE_EVM_RPC_URL") - proxyServiceHostname = os.Getenv("TEST_PROXY_SERVICE_EVM_RPC_HOSTNAME") - proxyServiceDataURL = os.Getenv("TEST_PROXY_SERVICE_EVM_RPC_DATA_URL") + proxyServiceURL = os.Getenv("TEST_PROXY_SERVICE_EVM_RPC_URL") + proxyServiceHostname = os.Getenv("TEST_PROXY_SERVICE_EVM_RPC_HOSTNAME") + proxyServicePruningURL = os.Getenv("TEST_PROXY_SERVICE_EVM_RPC_PRUNING_URL") + + proxyServiceHeightBasedRouting, _ = strconv.ParseBool(os.Getenv("TEST_PROXY_HEIGHT_BASED_ROUTING_ENABLED")) databaseURL = os.Getenv("TEST_DATABASE_ENDPOINT_URL") databasePassword = os.Getenv("DATABASE_PASSWORD") @@ -110,11 +118,11 @@ func TestE2ETestProxyProxiesForMultipleHosts(t *testing.T) { assert.Greater(t, int(header.Number.Int64()), 0) - dataClient, err := ethclient.Dial(proxyServiceDataURL) + pruningClient, err := ethclient.Dial(proxyServicePruningURL) require.NoError(t, err) - header, err = dataClient.HeaderByNumber(testContext, nil) + header, err = pruningClient.HeaderByNumber(testContext, nil) require.NoError(t, err) assert.Greater(t, int(header.Number.Int64()), 0) @@ -344,3 +352,86 @@ func TestE2ETestProxyTracksBlockNumberForMethodsWithBlockHashParam(t *testing.T) assert.Equal(t, *requestMetricDuringRequestWindow.BlockNumber, requestBlockNumber) } } + +func TestE2ETest_HeightBasedRouting(t *testing.T) { + if !proxyServiceHeightBasedRouting { + t.Skip("TEST_PROXY_HEIGHT_BASED_ROUTING_ENABLED is false. skipping height-based routing e2e test") + } + + rpc, err := rpc.Dial(proxyServiceURL) + require.NoError(t, err) + + databaseClient, err := database.NewPostgresClient(databaseConfig) + require.NoError(t, err) + + testCases := []struct { + name string + method string + params []interface{} + expectRoute string + }{ + { + name: "request for non-latest height -> default", + method: "eth_getBlockByNumber", + params: []interface{}{"0x2", false}, + expectRoute: service.ResponseBackendDefault, + }, + { + name: "request for earliest height -> default", + method: "eth_getBlockByNumber", + params: []interface{}{"earliest", false}, + expectRoute: service.ResponseBackendDefault, + }, + { + name: "request for latest height -> pruning", + method: "eth_getBlockByNumber", + params: []interface{}{"latest", false}, + expectRoute: service.ResponseBackendPruning, + }, + { + name: "request for finalized height -> pruning", + method: "eth_getBlockByNumber", + params: []interface{}{"finalized", false}, + expectRoute: service.ResponseBackendPruning, + }, + { + name: "request with empty height -> pruning", + method: "eth_getBlockByNumber", + params: []interface{}{nil, false}, + expectRoute: service.ResponseBackendPruning, + }, + { + name: "request not requiring height -> pruning", + method: "eth_chainId", + params: []interface{}{}, + expectRoute: service.ResponseBackendPruning, + }, + { + name: "request by hash -> default", + method: "eth_getBlockByHash", + params: []interface{}{"0xe9bd10bc1d62b4406dd1fb3dbf3adb54f640bdb9ebbe3dd6dfc6bcc059275e54", false}, + expectRoute: service.ResponseBackendDefault, + }, + { + name: "un-parseable (invalid) height -> default", + method: "eth_getBlockByNumber", + params: []interface{}{"not-a-block-tag", false}, + expectRoute: service.ResponseBackendDefault, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + startTime := time.Now() + err := rpc.Call(nil, tc.method, tc.params...) + require.NoError(t, err) + + metrics := findMetricsInWindowForMethods(databaseClient, startTime, time.Now(), []string{tc.method}) + + require.Len(t, metrics, 1) + fmt.Printf("%+v\n", metrics[0]) + require.Equal(t, metrics[0].MethodName, tc.method) + require.Equal(t, metrics[0].ResponseBackend, tc.expectRoute) + }) + } +} diff --git a/service/middleware.go b/service/middleware.go index eb06dbc..61da20a 100644 --- a/service/middleware.go +++ b/service/middleware.go @@ -28,6 +28,7 @@ const ( RequestUserAgentContextKey = "X-KAVA-PROXY-USER-AGENT" RequestRefererContextKey = "X-KAVA-PROXY-REFERER" RequestOriginContextKey = "X-KAVA-PROXY-ORIGIN" + ProxyMetadataContextKey = "X-KAVA-PROXY-RESPONSE-BACKEND" // Values defined by upstream services LoadBalancerForwardedForHeaderKey = "X-Forwarded-For" UserAgentHeaderkey = "User-Agent" @@ -167,7 +168,7 @@ func createProxyRequestMiddleware(next http.Handler, config config.Config, servi // proxy the request to the backend origin server // based on the request host - proxy, ok := proxies.ProxyForRequest(r) + proxy, proxyMetadata, ok := proxies.ProxyForRequest(r) if !ok { serviceLogger.Error().Msg(fmt.Sprintf("no matching proxy for host %s for request %+v\n configured proxies %+v", r.Host, r, proxies)) @@ -236,6 +237,9 @@ func createProxyRequestMiddleware(next http.Handler, config config.Config, servi enrichedContext := requestHostnameContext + // add response backend name to context + enrichedContext = context.WithValue(enrichedContext, ProxyMetadataContextKey, proxyMetadata) + // parse the remote address of the request for use below remoteAddressParts := strings.Split(r.RemoteAddr, ":") @@ -373,6 +377,14 @@ func createAfterProxyFinalizer(service *ProxyService, config config.Config) http return } + rawProxyMetadata := r.Context().Value(ProxyMetadataContextKey) + proxyMetadata, ok := rawProxyMetadata.(ProxyMetadata) + if !ok { + service.ServiceLogger.Trace().Msg(fmt.Sprintf("invalid context value %+v for value %s", proxyMetadata, ProxyMetadataContextKey)) + + return + } + var blockNumber *int64 rawBlockNumber, err := decodedRequestBody.ExtractBlockNumberFromEVMRPCRequest(r.Context(), service.evmClient) @@ -395,6 +407,8 @@ func createAfterProxyFinalizer(service *ProxyService, config config.Config) http Referer: &referer, Origin: &origin, BlockNumber: blockNumber, + ResponseBackend: proxyMetadata.BackendName, + ResponseBackendRoute: proxyMetadata.BackendRoute.String(), } // save metric to database diff --git a/service/proxy.go b/service/proxy.go index 2e86007..ca81b95 100644 --- a/service/proxy.go +++ b/service/proxy.go @@ -4,49 +4,77 @@ import ( "fmt" "net/http" "net/http/httputil" + "net/url" "github.com/kava-labs/kava-proxy-service/config" "github.com/kava-labs/kava-proxy-service/logging" ) +// ResponseBackend values for metric reporting of where request was routed +const ( + ResponseBackendDefault = "DEFAULT" + ResponseBackendPruning = "PRUNING" +) + // Proxies is an interface for getting a reverse proxy for a given request. +// proxy is the reverse proxy to use for the request type Proxies interface { - ProxyForRequest(r *http.Request) (proxy *httputil.ReverseProxy, found bool) + ProxyForRequest(r *http.Request) (proxy *httputil.ReverseProxy, metadata ProxyMetadata, found bool) +} + +// ProxyMetadata wraps details about the proxy used for a request. +// It is useful for gathering details about the proxied request to include in metrics. +type ProxyMetadata struct { + // name of the backend used + BackendName string + // url of the backend used + BackendRoute url.URL } // NewProxies creates a Proxies instance based on the service configuration: // - for non-sharding configuration, it returns a HostProxies -// TODO: - for sharding configurations, it returns a HeightShardingProxies +// - for sharding configurations, it returns a HeightShardingProxies func NewProxies(config config.Config, serviceLogger *logging.ServiceLogger) Proxies { + if config.EnableHeightBasedRouting { + serviceLogger.Debug().Msg("configuring reverse proxies based on host AND height") + return newHeightShardingProxies(config, serviceLogger) + } serviceLogger.Debug().Msg("configuring reverse proxies based solely on request host") - return newHostProxies(config, serviceLogger) + return newHostProxies(ResponseBackendDefault, config.ProxyBackendHostURLMapParsed, serviceLogger) } // HostProxies chooses a proxy based solely on the Host of the incoming request, // and the host -> backend url map defined in the config. +// HostProxies name is the response backend provided for all requests type HostProxies struct { - proxyForHost map[string]*httputil.ReverseProxy + name string + proxyForHost map[string]*httputil.ReverseProxy + targetUrlForHost map[string]url.URL } var _ Proxies = HostProxies{} // ProxyForRequest implements Proxies. It determines the proxy based solely on the request Host. -func (hbp HostProxies) ProxyForRequest(r *http.Request) (*httputil.ReverseProxy, bool) { +func (hbp HostProxies) ProxyForRequest(r *http.Request) (*httputil.ReverseProxy, ProxyMetadata, bool) { proxy, found := hbp.proxyForHost[r.Host] - return proxy, found + metadata := ProxyMetadata{ + BackendName: hbp.name, + BackendRoute: hbp.targetUrlForHost[r.Host], + } + return proxy, metadata, found } // newHostProxies creates a HostProxies from the backend url map defined in the config. -func newHostProxies(config config.Config, serviceLogger *logging.ServiceLogger) HostProxies { +func newHostProxies(name string, hostURLMap map[string]url.URL, serviceLogger *logging.ServiceLogger) HostProxies { reverseProxyForHost := make(map[string]*httputil.ReverseProxy) - for host, proxyBackendURL := range config.ProxyBackendHostURLMapParsed { + for host, proxyBackendURL := range hostURLMap { serviceLogger.Debug().Msg(fmt.Sprintf("creating reverse proxy for host %s to %+v", host, proxyBackendURL)) - targetURL := config.ProxyBackendHostURLMapParsed[host] + targetURL := hostURLMap[host] reverseProxyForHost[host] = httputil.NewSingleHostReverseProxy(&targetURL) } - return HostProxies{proxyForHost: reverseProxyForHost} + return HostProxies{name: name, proxyForHost: reverseProxyForHost, targetUrlForHost: hostURLMap} } diff --git a/service/proxy_test.go b/service/proxy_test.go index f9bd6a3..15022bf 100644 --- a/service/proxy_test.go +++ b/service/proxy_test.go @@ -1,6 +1,7 @@ package service_test import ( + "context" "fmt" "net/http" "net/http/httputil" @@ -8,53 +9,70 @@ import ( "testing" "github.com/kava-labs/kava-proxy-service/config" + "github.com/kava-labs/kava-proxy-service/decode" "github.com/kava-labs/kava-proxy-service/service" "github.com/stretchr/testify/require" ) -func newConfig(t *testing.T, rawHostMap string) config.Config { - parsed, err := config.ParseRawProxyBackendHostURLMap(rawHostMap) +func newConfig(t *testing.T, defaultHostMap string, pruningHostMap string) config.Config { + parsed, err := config.ParseRawProxyBackendHostURLMap(defaultHostMap) require.NoError(t, err) - return config.Config{ - ProxyBackendHostURLMapRaw: rawHostMap, + result := config.Config{ + ProxyBackendHostURLMapRaw: defaultHostMap, ProxyBackendHostURLMapParsed: parsed, } + if pruningHostMap != "" { + result.EnableHeightBasedRouting = true + result.ProxyPruningBackendHostURLMapRaw = pruningHostMap + result.ProxyPruningBackendHostURLMap, err = config.ParseRawProxyBackendHostURLMap(pruningHostMap) + require.NoError(t, err) + } + return result } func TestUnitTest_NewProxies(t *testing.T) { t.Run("returns a HostProxies when sharding disabled", func(t *testing.T) { - config := newConfig(t, dummyConfig.ProxyBackendHostURLMapRaw) + config := newConfig(t, dummyConfig.ProxyBackendHostURLMapRaw, "") proxies := service.NewProxies(config, dummyLogger) require.IsType(t, service.HostProxies{}, proxies) }) - // TODO: HeightShardingProxies + + t.Run("returns a HeightShardingProxies when sharding enabled", func(t *testing.T) { + config := newConfig(t, dummyConfig.ProxyBackendHostURLMapRaw, dummyConfig.ProxyPruningBackendHostURLMapRaw) + proxies := service.NewProxies(config, dummyLogger) + require.IsType(t, service.HeightShardingProxies{}, proxies) + }) } func TestUnitTest_HostProxies(t *testing.T) { config := newConfig(t, "magic.kava.io>magicalbackend.kava.io,archive.kava.io>archivenode.kava.io,pruning.kava.io>pruningnode.kava.io", + "", ) proxies := service.NewProxies(config, dummyLogger) t.Run("ProxyForHost maps to correct proxy", func(t *testing.T) { req := mockReqForUrl("//magic.kava.io") - proxy, found := proxies.ProxyForRequest(req) + proxy, metadata, found := proxies.ProxyForRequest(req) require.True(t, found, "expected proxy to be found") + require.Equal(t, metadata.BackendName, service.ResponseBackendDefault) requireProxyRoutesToUrl(t, proxy, req, "magicalbackend.kava.io/") req = mockReqForUrl("https://archive.kava.io") - proxy, found = proxies.ProxyForRequest(req) + proxy, metadata, found = proxies.ProxyForRequest(req) require.True(t, found, "expected proxy to be found") + require.Equal(t, metadata.BackendName, service.ResponseBackendDefault) requireProxyRoutesToUrl(t, proxy, req, "archivenode.kava.io/") req = mockReqForUrl("//pruning.kava.io/some/nested/endpoint") - proxy, found = proxies.ProxyForRequest(req) + proxy, metadata, found = proxies.ProxyForRequest(req) require.True(t, found, "expected proxy to be found") + require.Equal(t, metadata.BackendName, service.ResponseBackendDefault) requireProxyRoutesToUrl(t, proxy, req, "pruningnode.kava.io/some/nested/endpoint") }) t.Run("ProxyForHost fails with unknown host", func(t *testing.T) { - _, found := proxies.ProxyForRequest(mockReqForUrl("//unknown-host.kava.io")) + _, _, found := proxies.ProxyForRequest(mockReqForUrl("//unknown-host.kava.io")) require.False(t, found, "expected proxy not found for unknown host") }) } @@ -71,6 +89,16 @@ func mockReqForUrl(reqUrl string) *http.Request { return &http.Request{Host: parsed.Host, URL: parsed} } +func mockJsonRpcReqToUrl(url string, evmReq *decode.EVMRPCRequestEnvelope) *http.Request { + req := mockReqForUrl(url) + // add the request into the request context, mocking previous middleware parsing + ctx := context.Background() + if evmReq != nil { + ctx = context.WithValue(ctx, service.DecodedRequestContextKey, evmReq) + } + return req.WithContext(ctx) +} + // requireProxyRoutesToUrl is a test helper that verifies that // the given proxy maps the provided request to the expected proxy backend // relies on the fact that reverse proxies are given a Director that rewrite the request's URL diff --git a/service/service_test.go b/service/service_test.go index c0120cb..9733509 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -12,31 +12,38 @@ import ( ) var ( - testDefaultContext = context.TODO() - proxyServiceURLMapRaw = os.Getenv("TEST_PROXY_BACKEND_HOST_URL_MAP") - databaseName = os.Getenv("DATABASE_NAME") - databaseUsername = os.Getenv("DATABASE_USERNAME") - databasePassword = os.Getenv("DATABASE_PASSWORD") - databaseEndpointURL = os.Getenv("DATABASE_ENDPOINT_URL") - testServiceLogLevel = os.Getenv("TEST_SERVICE_LOG_LEVEL") - evmQueryServiceURL = os.Getenv("TEST_EVM_QUERY_SERVICE_URL") + testDefaultContext = context.TODO() + proxyServiceDefaultURLMapRaw = os.Getenv("TEST_PROXY_BACKEND_HOST_URL_MAP") + proxyServicePruningURLMapRaw = os.Getenv("TEST_PROXY_PRUNING_BACKEND_HOST_URL_MAP") + databaseName = os.Getenv("DATABASE_NAME") + databaseUsername = os.Getenv("DATABASE_USERNAME") + databasePassword = os.Getenv("DATABASE_PASSWORD") + databaseEndpointURL = os.Getenv("DATABASE_ENDPOINT_URL") + testServiceLogLevel = os.Getenv("TEST_SERVICE_LOG_LEVEL") + evmQueryServiceURL = os.Getenv("TEST_EVM_QUERY_SERVICE_URL") dummyConfig = func() config.Config { - proxyBackendHostURLMapParsed, err := config.ParseRawProxyBackendHostURLMap(proxyServiceURLMapRaw) - + proxyBackendHostURLMapParsed, err := config.ParseRawProxyBackendHostURLMap(proxyServiceDefaultURLMapRaw) + if err != nil { + panic(err) + } + proxyPruningBackendHostURLMapParsed, err := config.ParseRawProxyBackendHostURLMap(proxyServicePruningURLMapRaw) if err != nil { panic(err) } conf := config.Config{ - ProxyBackendHostURLMapRaw: proxyServiceURLMapRaw, - ProxyBackendHostURLMapParsed: proxyBackendHostURLMapParsed, - DatabaseName: databaseName, - DatabaseUserName: databaseUsername, - DatabasePassword: databasePassword, - DatabaseEndpointURL: databaseEndpointURL, - EvmQueryServiceURL: evmQueryServiceURL, + ProxyBackendHostURLMapRaw: proxyServiceDefaultURLMapRaw, + ProxyBackendHostURLMapParsed: proxyBackendHostURLMapParsed, + ProxyPruningBackendHostURLMapRaw: proxyServicePruningURLMapRaw, + ProxyPruningBackendHostURLMap: proxyPruningBackendHostURLMapParsed, + + DatabaseName: databaseName, + DatabaseUserName: databaseUsername, + DatabasePassword: databasePassword, + DatabaseEndpointURL: databaseEndpointURL, + EvmQueryServiceURL: evmQueryServiceURL, } return conf diff --git a/service/shard.go b/service/shard.go new file mode 100644 index 0000000..441068f --- /dev/null +++ b/service/shard.go @@ -0,0 +1,96 @@ +package service + +import ( + "fmt" + "net/http" + "net/http/httputil" + + "github.com/kava-labs/kava-proxy-service/config" + "github.com/kava-labs/kava-proxy-service/decode" + "github.com/kava-labs/kava-proxy-service/logging" +) + +// HeightShardingProxies routes traffic based on the host _and_ the height of the query. +// If the height is "latest" (or equivalent), return Pruning node proxy host. +// Otherwise return default node proxy host. +type HeightShardingProxies struct { + *logging.ServiceLogger + + pruningProxies HostProxies + defaultProxies HostProxies +} + +var _ Proxies = HeightShardingProxies{} + +// ProxyForRequest implements Proxies. +// Decodes height of request +// - routes to Pruning proxy if defined & height is "latest" +// - otherwise routes to Default proxy +func (hsp HeightShardingProxies) ProxyForRequest(r *http.Request) (*httputil.ReverseProxy, ProxyMetadata, bool) { + _, _, found := hsp.pruningProxies.ProxyForRequest(r) + // if the host isn't in the pruning proxies, short circuit fallback to default + if !found { + hsp.Debug().Msg(fmt.Sprintf("no pruning host backend configured for %s", r.Host)) + return hsp.defaultProxies.ProxyForRequest(r) + } + + // parse the height of the request + req := r.Context().Value(DecodedRequestContextKey) + decodedReq, ok := (req).(*decode.EVMRPCRequestEnvelope) + if !ok { + hsp.Error().Msg("HeightShardingProxies failed to find & cast the decoded request envelope from the request context") + return hsp.defaultProxies.ProxyForRequest(r) + } + + // some RPC methods can always be routed to the latest block + if decode.MethodRequiresNoHistory(decodedReq.Method) { + hsp.Debug().Msg(fmt.Sprintf("request method %s can always use latest block. routing to pruning proxy", decodedReq.Method)) + return hsp.pruningProxies.ProxyForRequest(r) + } + + // short circuit if requesting a method that doesn't include block height number + if !decode.MethodHasBlockNumberParam(decodedReq.Method) { + hsp.Debug().Msg(fmt.Sprintf("request method does not include block height (%s). routing to default proxy", decodedReq.Method)) + return hsp.defaultProxies.ProxyForRequest(r) + } + + // parse height from the request + height, err := decode.ParseBlockNumberFromParams(decodedReq.Method, decodedReq.Params) + if err != nil { + hsp.Error().Msg(fmt.Sprintf("expected but failed to parse block number for %+v: %s", decodedReq, err)) + return hsp.defaultProxies.ProxyForRequest(r) + } + + // route "latest" to pruning proxy, otherwise route to default + if shouldRouteToPruning(height) { + hsp.Debug().Msg(fmt.Sprintf("request is for latest height (%d). routing to pruning proxy", height)) + return hsp.pruningProxies.ProxyForRequest(r) + } + hsp.Debug().Msg(fmt.Sprintf("request is for specific height (%d). routing to default proxy", height)) + return hsp.defaultProxies.ProxyForRequest(r) +} + +// newHeightShardingProxies creates a new HeightShardingProxies from the service config. +func newHeightShardingProxies(config config.Config, serviceLogger *logging.ServiceLogger) HeightShardingProxies { + return HeightShardingProxies{ + ServiceLogger: serviceLogger, + pruningProxies: newHostProxies(ResponseBackendPruning, config.ProxyPruningBackendHostURLMap, serviceLogger), + defaultProxies: newHostProxies(ResponseBackendDefault, config.ProxyBackendHostURLMapParsed, serviceLogger), + } +} + +// lookup map for block tags that all represent "latest". +// maps encoded block tag -> true if the block tag should route to pruning cluster. +var blockTagEncodingsRoutedToLatest = map[int64]bool{ + decode.BlockTagToNumberCodec[decode.BlockTagLatest]: true, + decode.BlockTagToNumberCodec[decode.BlockTagFinalized]: true, + decode.BlockTagToNumberCodec[decode.BlockTagPending]: true, + decode.BlockTagToNumberCodec[decode.BlockTagSafe]: true, + decode.BlockTagToNumberCodec[decode.BlockTagEmpty]: true, +} + +// shouldRouteToPruning is a helper method for determining if an encoded block tag should get routed +// to the pruning cluster +func shouldRouteToPruning(encodedHeight int64) bool { + return blockTagEncodingsRoutedToLatest[encodedHeight] +} diff --git a/service/shard_test.go b/service/shard_test.go new file mode 100644 index 0000000..2f80405 --- /dev/null +++ b/service/shard_test.go @@ -0,0 +1,162 @@ +package service_test + +import ( + "fmt" + "testing" + + "github.com/kava-labs/kava-proxy-service/decode" + "github.com/kava-labs/kava-proxy-service/service" + "github.com/stretchr/testify/require" +) + +func TestUnitTest_HeightShardingProxies(t *testing.T) { + archiveBackend := "archivenode.kava.io/" + pruningBackend := "pruningnode.kava.io/" + config := newConfig(t, + fmt.Sprintf("archive.kava.io>%s,pruning.kava.io>%s", archiveBackend, pruningBackend), + fmt.Sprintf("archive.kava.io>%s", pruningBackend), + ) + proxies := service.NewProxies(config, dummyLogger) + + testCases := []struct { + name string + url string + req *decode.EVMRPCRequestEnvelope + expectFound bool + expectBackend string + expectRoute string + }{ + // DEFAULT ROUTE CASES + { + name: "routes to default when not in pruning map", + url: "//pruning.kava.io", + req: &decode.EVMRPCRequestEnvelope{}, + expectFound: true, + expectBackend: service.ResponseBackendDefault, + expectRoute: pruningBackend, + }, + { + name: "routes to default for specific non-latest height", + url: "//archive.kava.io", + req: &decode.EVMRPCRequestEnvelope{ + Method: "eth_getBlockByNumber", + Params: []interface{}{"0xbaddad", false}, + }, + expectFound: true, + expectBackend: service.ResponseBackendDefault, + expectRoute: archiveBackend, + }, + { + name: "routes to default for methods that don't have block number", + url: "//archive.kava.io", + req: &decode.EVMRPCRequestEnvelope{ + Method: "eth_getBlockByHash", + Params: []interface{}{"0xe9bd10bc1d62b4406dd1fb3dbf3adb54f640bdb9ebbe3dd6dfc6bcc059275e54", false}, + }, + expectFound: true, + expectBackend: service.ResponseBackendDefault, + expectRoute: archiveBackend, + }, + { + name: "routes to default if it fails to decode req", + url: "//archive.kava.io", + req: nil, + expectFound: true, + expectBackend: service.ResponseBackendDefault, + expectRoute: archiveBackend, + }, + { + name: "routes to default if it fails to parse block number", + url: "//archive.kava.io", + req: &decode.EVMRPCRequestEnvelope{ + Method: "eth_getBlockByNumber", + Params: []interface{}{"not-a-block-tag", false}, + }, + expectFound: true, + expectBackend: service.ResponseBackendDefault, + expectRoute: archiveBackend, + }, + { + name: "routes to default for 'earliest' block", + url: "//archive.kava.io", + req: &decode.EVMRPCRequestEnvelope{ + Method: "eth_getBlockByNumber", + Params: []interface{}{"earliest", false}, + }, + expectFound: true, + expectBackend: service.ResponseBackendDefault, + expectRoute: archiveBackend, + }, + + // PRUNING ROUTE CASES + { + name: "routes to pruning for 'latest' block", + url: "//archive.kava.io", + req: &decode.EVMRPCRequestEnvelope{ + Method: "eth_getBlockByNumber", + Params: []interface{}{"latest", false}, + }, + expectFound: true, + expectBackend: service.ResponseBackendPruning, + expectRoute: pruningBackend, + }, + { + name: "routes to pruning when block number empty", + url: "//archive.kava.io", + req: &decode.EVMRPCRequestEnvelope{ + Method: "eth_getBlockByNumber", + Params: []interface{}{nil, false}, + }, + expectFound: true, + expectBackend: service.ResponseBackendPruning, + expectRoute: pruningBackend, + }, + { + name: "routes to pruning for no-history methods", + url: "//archive.kava.io", + req: &decode.EVMRPCRequestEnvelope{ + Method: "eth_chainId", + }, + expectFound: true, + expectBackend: service.ResponseBackendPruning, + expectRoute: pruningBackend, + }, + { + // this is just another example of the above, but worth pointing out! + name: "routes to pruning when sending txs", + url: "//archive.kava.io", + req: &decode.EVMRPCRequestEnvelope{ + Method: "eth_sendTransaction", + Params: []interface{}{ + map[string]string{ + "from": "0xdeadbeef00000000000000000000000000000123", + "to": "0xbaddad0000000000000000000000000000000123", + "value": "0x1", + "gas": "0xeeee", + "gasPrice": "0x12345678900", + "nonce": "0x0", + }, + }, + }, + expectFound: true, + expectBackend: service.ResponseBackendPruning, + expectRoute: pruningBackend, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + req := mockJsonRpcReqToUrl(tc.url, tc.req) + proxy, metadata, found := proxies.ProxyForRequest(req) + if !tc.expectFound { + require.False(t, found, "expected proxy not to be found") + return + } + require.True(t, found, "expected proxy to be found") + require.NotNil(t, proxy) + require.Equal(t, metadata.BackendName, tc.expectBackend) + require.Equal(t, metadata.BackendRoute.String(), tc.expectRoute) + requireProxyRoutesToUrl(t, proxy, req, tc.expectRoute) + }) + } +}