add pruning vs archive node sharding (#42)

* add height-based routing options to config

PROXY_HEIGHT_BASED_ROUTING_ENABLED enables the feature
PROXY_PRUNING_BACKEND_HOST_URL_MAP is the map to pruning cluster hosts

* add HeightShardingProxies for height-based routing

the Proxies instance parses the block height out of the request.
if the request is made for the latest block height, the request is routed
to the defined pruning cluster. otherwise the default cluster is used.

* support & handle routing always-latest methods

some methods will always work if routed to a pruning cluster. do that!

* handle routing block tags to pruning cluster

* refactor & rename NoHistoryMethods

* refactor method param type checks to pure functions

* add eth_blockNumber to no-history methods

* add unit tests for HeightShardingProxies

* add proxy routing documentation

* add ErrEmptyHostMap to allow handling of that error

* allow empty pruning host map

* ensure pruning host map are all in default host map

* rename "data" test client to "pruning"

* add named ResponseBackend concept to Proxies

* add ResponseBackend to proxied_request_metrics

* use HeaderByHash for fewer queries

* add e2e tests for HeightBasedRouting

* fix proxy test for expected routes

* rename PROXY_CONTAINER_EVM_RPC_DATA_PORT -> PRUNING

* update CI e2e test configuration

* add more sharding proxy docs & docs on metrics

* add ci-setup w/ docs for local CI testnet test runs

* more docs context

* reorganize proxy code into shard.go

* track actual url the requests are routed to

.env

@@ -27,18 +27,20 @@ KAVA_PRUNING_HOST_EVM_RPC_PORT=8555
 KAVA_PRUNING_HOST_COSMOS_RPC_PORT=26667
 
 PROXY_CONTAINER_PORT=7777
-PROXY_CONTAINER_EVM_RPC_DATA_PORT=7778
+PROXY_CONTAINER_EVM_RPC_PRUNING_PORT=7778
 PROXY_HOST_PORT=7777
 PROXY_CONTAINER_DEBUG_PORT=2345
 PROXY_HOST_DEBUG_PORT=2345
 
 ##### E2E Testing Config
 TEST_PROXY_SERVICE_EVM_RPC_URL=http://localhost:7777
 TEST_PROXY_SERVICE_EVM_RPC_HOSTNAME=localhost:7777
-TEST_PROXY_SERVICE_EVM_RPC_DATA_URL=http://localhost:7778
+TEST_PROXY_SERVICE_EVM_RPC_PRUNING_URL=http://localhost:7778
 TEST_PROXY_BACKEND_EVM_RPC_HOST_URL=http://localhost:8545
 TEST_DATABASE_ENDPOINT_URL=localhost:5432
 TEST_PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-validator:8545,localhost:7778>http://kava-pruning:8545
+TEST_PROXY_HEIGHT_BASED_ROUTING_ENABLED=true
+TEST_PROXY_PRUNING_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-pruning:8545,localhost:7778>http://kava-pruning:8545
 # What level of logging to use for service objects constructed during
 # unit tests
 TEST_SERVICE_LOG_LEVEL=ERROR
@@ -57,6 +59,11 @@ HTTP_READ_TIMEOUT_SECONDS=30
 HTTP_WRITE_TIMEOUT_SECONDS=60
 # Address of the origin server to proxy all requests to
 PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-validator:8545,localhost:7778>http://kava-pruning:8545
+# height-based routing will look at the height of an incoming EVM request
+# iff. the height is "latest", it routes to the corresponding PROXY_PRUNING_BACKEND_HOST_URL_MAP value
+# otherwise, it falls back to the value in PROXY_BACKEND_HOST_URL_MAP
+PROXY_HEIGHT_BASED_ROUTING_ENABLED=true
+PROXY_PRUNING_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-pruning:8545,localhost:7778>http://kava-pruning:8545
 # Configuration for the servcie to connect to it's database
 DATABASE_NAME=postgres
 DATABASE_ENDPOINT_URL=postgres:5432

DEVELOPMENT.md

@@ -98,6 +98,17 @@ make ready e2e-test
 
 For details on the local E2E setup, see [the `docker` directory](./docker/README.md).
 
+#### Against testnet
+
+The Continuous Integration (CI) for this repo is setup to run a local proxy service with database & redis, but configures the service to use public testnet urls for the kava requests. This allows for testing the proxy service against a production-like environment (requests are routed to public testnet).
+
+You can emulate the CI configuration in your local environment:
+```bash
+make ci-setup
+```
+
+At that point, running `make e2e-test` will run the end-to-end tests with requests routing to public testnet.
+
 ## Test Coverage Report
 
 The test commands `make test`, `make unit-test`, and `make e2e-test` generate a `cover.out` raw test coverage report. The coverage can be converted into a user-friendly webpage:

Makefile

@@ -44,6 +44,14 @@ unit-test:
 e2e-test:
 	go test -count=1 -v -cover -coverprofile cover.out --race ./... -run "^TestE2ETest*"
 
+.PHONY: ci-setup
+# set up your local environment such that running `make e2e-test` runs against testnet (like in CI)
+ci-setup:
+	docker compose -f ci.docker-compose.yml pull
+	docker compose -f ci.docker-compose.yml up -d --build --force-recreate
+	PROXY_CONTAINER_PORT=7777 bash scripts/wait-for-proxy-service-running.sh
+	PROXY_CONTAINER_PORT=7777 MINIMUM_REQUIRED_PARTITIONS=30 bash scripts/wait-for-proxy-service-running.sh
+
 .PHONY: it
 # run any test matching the provided pattern, can pass a regex or a string
 # of the exact test to run

architecture/MIDDLEWARE.MD

@@ -61,27 +61,7 @@ server := &http.Server{
 
 1. Times the roundtrip latency for the response from the backend origin server and stores the latency in the context key `X-KAVA-PROXY-ORIGIN-ROUNDTRIP-LATENCY-MILLISECONDS` for use by other middleware.
 
-#### Configuration
-
-The proxy is configured to route requests based on their incoming Host. These are controlled via the
-`PROXY_BACKEND_HOST_URL_MAP` environment variable.
-
-As an example, consider a value of `localhost:7777>http://kava:8545,localhost:7778>http://kava:8545`.
-This value is parsed into a map that looks like the following:
-```
-{
-    "localhost:7777" => "http://kava:8545",
-    "localhost:7778" => "http://kava:8545",
-}
-```
-Any request to the service will be routed according to this map.
-ie. all requests to local ports 7777 & 7778 get forwarded to `http://kava:8545`
-
-Implementations of the [`Proxies` interface](../service/proxy.go#L13) contain logic for deciding
-the backend host url to which a request is routed. This is used in the ProxyRequestMiddleware to
-route requests.
-
-Any request made to a host not in the map responds 502 Bad Gateway.
+See [Proxy Routing](./PROXY_ROUTING.md) for details on configuration and how requests are routed.
 
 ### After Proxy Middleware
 

architecture/PROXY_ROUTING.md

@@ -0,0 +1,152 @@
+# Proxy Routing
+
+The proxy chooses where to route a request primarily by the incoming Host URL to which the client
+originally made their request. The routing is configured by maps of the Host to a backend url in
+the environment variables.
+
+All possible configurations use the `PROXY_BACKEND_HOST_URL_MAP` environment variable. This encodes
+the default backend to route all requests from a given host. Additional functionality is available
+via the `PROXY_HEIGHT_BASED_ROUTING_ENABLED` env variable (see [Rudimentary Sharding](#rudimentary-sharding)).
+
+Consider the simplest case: a host-based-only routing proxy configured for one host
+```
+PROXY_HEIGHT_BASED_ROUTING_ENABLED=false
+PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava:8545
+```
+This value is parsed into a map that looks like the following:
+```
+{
+    "localhost:7777" => "http://kava:8545",
+}
+```
+Any request to the service will be routed according to this map.
+ie. all requests to local port 7777 get forwarded to `http://kava:8545`
+
+Implementations of the [`Proxies` interface](../service/proxy.go#L13) contain logic for deciding
+the backend host url to which a request is routed. This is used in the ProxyRequestMiddleware to
+route requests.
+
+Any request made to a host not in the map responds 502 Bad Gateway.
+
+## More Examples of Host-only routing
+
+Here is a diagram of the above network topology:
+
+![Proxy Service configured for one host](images/proxy_service_simple_one_host.jpg)
+
+In this simple configuration of only having default hosts, you can do many things:
+
+**Many hosts -> One backend**
+
+```
+PROXY_HEIGHT_BASED_ROUTING_ENABLED=false
+PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava:8545,localhost:7778>http://kava:8545
+```
+This value is parsed into a map that looks like the following:
+```
+{
+    "localhost:7777" => "http://kava:8545",
+    "localhost:7778" => "http://kava:8545",
+}
+```
+All requests to local ports 7777 & 7778 route to the same cluster at kava:8545
+
+![Proxy Service configured for many hosts for one backend](images/proxy_service_many_hosts_one_backend.jpg)
+
+**Many hosts -> Many backends**
+
+```
+PROXY_HEIGHT_BASED_ROUTING_ENABLED=false
+PROXY_BACKEND_HOST_URL_MAP=evm.kava.io>http://kava-pruning:8545,evm.data.kava.io>http://kava-archive:8545
+```
+This value is parsed into a map that looks like the following:
+```
+{
+    "evm.kava.io"      => "http://kava-pruning:8545",
+    "evm.data.kava.io" => "http://kava-archive:8545",
+}
+```
+Requests made to evm.kava.io route to a pruning cluster.
+Those made to evm.data.kava.io route to an archive cluster.
+
+![Proxy Service configured for many hosts with many backends](images/proxy_service_many_hosts_many_backends.jpg)
+
+## Rudimentary Sharding
+
+Now suppose you want multiple backends for the same host.
+
+The proxy service supports height-based routing to direct requests that only require the most recent
+block to a different cluster.
+This support is handled via the [`HeightShardingProxies` implementation](../service/shard.go#L16).
+
+This is configured via the `PROXY_HEIGHT_BASED_ROUTING_ENABLED` and `PROXY_PRUNING_BACKEND_HOST_URL_MAP`
+environment variables.
+* `PROXY_HEIGHT_BASED_ROUTING_ENABLED` - flag to toggle this functionality
+* `PROXY_PRUNING_BACKEND_HOST_URL_MAP` - like `PROXY_BACKEND_HOST_URL_MAP`, but only used for JSON-RPC
+  requests that target the latest block (or are stateless, like `eth_chainId`, `eth_coinbase`, etc).
+
+For example, to lighten the load for your resource-intensive (& expensive) archive cluster, you can
+route all requests for the "latest" block to a less resource-intensive (& cheaper) pruning cluster:
+```
+PROXY_HEIGHT_BASED_ROUTING_ENABLED=true
+PROXY_BACKEND_HOST_URL_MAP=evm.data.kava.io>http://kava-archive:8545
+PROXY_PRUNING_BACKEND_HOST_URL_MAP=evm.data.kava.io>http://kava-pruning:8545
+```
+This value is parsed into a map that looks like the following:
+```
+{
+  "default": {
+    "evm.data.kava.io" => "http://kava-archive:8545",
+  },
+  "pruning": {
+    "evm.data.kava.io" => "http://kava-pruning:8545",
+  }
+}
+```
+All traffic to evm.data.kava.io that targets the latest block (or requires no history) routes to the pruning cluster.
+Otherwise, all traffic is sent to the archive cluster.
+
+![Proxy Service configured with rudimentary sharding](images/proxy_service_rudimentary_sharding.jpg)
+
+### Default vs Pruning Backend Routing
+
+When `PROXY_HEIGHT_BASED_ROUTING_ENABLED` is `true`, the following cases will cause requests to route
+to the the backend url defined in `PROXY_PRUNING_BACKEND_HOST_URL_MAP` (if present):
+* requests that include any of the following block tags:
+  * `"latest"`
+  * `"finalized"`
+  * `"pending"`
+  * `"safe"`
+  * empty/missing block tag (interpreted as `"latest"`)
+* requests for methods that require no historic state, including transaction broadcasting
+  * for a full list of methods, see [`NoHistoryMethods`](../decode/evm_rpc.go#L89)
+
+All other requests fallback to the default backend url defined in `PROXY_BACKEND_HOST_URL_MAP`.
+This includes
+* requests for hosts not included in `PROXY_PRUNING_BACKEND_HOST_URL_MAP`
+* requests targeting any specific height by number
+  * NOTE: the service does not track the current height of the chain. if the tip of the chain is at
+    block 1000, a query for block 1000 will still route to the default (not pruning) backend
+* requests for methods that use block hash, like `eth_getBlockByHash`
+* requests with unparsable (invalid) block numbers
+* requests for block tag `"earliest"`
+
+The service will panic on startup if a host in `PROXY_PRUNING_BACKEND_HOST_URL_MAP` is not present
+in `PROXY_BACKEND_HOST_URL_MAP`.
+
+Any request made to a host not in the `PROXY_BACKEND_HOST_URL_MAP` map responds 502 Bad Gateway.
+
+## Metrics
+
+When metrics are enabled, the `proxied_request_metrics` table tracks the backend to which requests
+are routed in the `response_backend` column.
+
+When height-based sharding is disabled (`PROXY_HEIGHT_BASED_ROUTING_ENABLED=false`), the value is
+always `DEFAULT`.
+
+When enabled, the column will have one of the following values:
+* `DEFAULT` - the request was routed to the backend defined in `PROXY_BACKEND_HOST_URL_MAP`
+* `PRUNING` - the request was routed to the backend defined in `PROXY_PRUNING_BACKEND_HOST_URL_MAP`
+
+Additionally, the actual URL to which the request is routed to is tracked in the
+`response_backend_route` column.

ci.docker-compose.yml

@@ -22,12 +22,14 @@ services:
       dockerfile: ci.Dockerfile
     env_file: .env
     environment:
+      PROXY_HEIGHT_BASED_ROUTING_ENABLED: true
       # use public testnet as backend origin server to avoid having
       # to self-host a beefy Github Action runner
       # to build and run a kava node each execution
-      PROXY_BACKEND_HOST_URL_MAP: localhost:7777>https://evmrpc.internal.testnet.proxy.kava.io,localhost:7778>https://evmrpcdata.internal.testnet.proxy.kava.io
+      PROXY_BACKEND_HOST_URL_MAP: localhost:7777>https://evmrpcdata.internal.testnet.proxy.kava.io,localhost:7778>https://evmrpc.internal.testnet.proxy.kava.io
+      PROXY_PRUNING_BACKEND_HOST_URL_MAP: localhost:7777>https://evmrpc.internal.testnet.proxy.kava.io
       EVM_QUERY_SERVICE_URL: https://evmrpc.internal.testnet.proxy.kava.io
     ports:
       - "${PROXY_HOST_PORT}:${PROXY_CONTAINER_PORT}"
-      - "${PROXY_CONTAINER_EVM_RPC_DATA_PORT}:${PROXY_CONTAINER_PORT}"
+      - "${PROXY_CONTAINER_EVM_RPC_PRUNING_PORT}:${PROXY_CONTAINER_PORT}"
       - "${PROXY_HOST_DEBUG_PORT}:${PROXY_CONTAINER_DEBUG_PORT}"

clients/database/migrations/20231013122742_add_response_backend.up.sql

@@ -0,0 +1,9 @@
+-- add response backend column, backfilling with "DEFAULT" (the only value to exist up until now)
+-- new metrics that omit its value are assumed to have been routed to "DEFAULT" backend
+-- also add response backend route. this is the backend url the request was routed to.
+ALTER TABLE
+  IF EXISTS proxied_request_metrics
+ADD
+  response_backend character varying DEFAULT 'DEFAULT',
+ADD
+  response_backend_route character varying;

clients/database/request_metric.go

@@ -27,6 +27,8 @@ type ProxiedRequestMetric struct {
 	UserAgent                   *string
 	Referer                     *string
 	Origin                      *string
+	ResponseBackend             string
+	ResponseBackendRoute        string
 }
 
 // Save saves the current ProxiedRequestMetric to

config/config.go

@@ -17,6 +17,9 @@ type Config struct {
 	LogLevel                               string
 	ProxyBackendHostURLMapRaw              string
 	ProxyBackendHostURLMapParsed           map[string]url.URL
+	EnableHeightBasedRouting               bool
+	ProxyPruningBackendHostURLMapRaw       string
+	ProxyPruningBackendHostURLMap          map[string]url.URL
 	EvmQueryServiceURL                     string
 	DatabaseName                           string
 	DatabaseEndpointURL                    string
@@ -43,6 +46,8 @@ const (
 	LOG_LEVEL_ENVIRONMENT_KEY                          = "LOG_LEVEL"
 	DEFAULT_LOG_LEVEL                                  = "INFO"
 	PROXY_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY         = "PROXY_BACKEND_HOST_URL_MAP"
+	PROXY_HEIGHT_BASED_ROUTING_ENABLED_KEY             = "PROXY_HEIGHT_BASED_ROUTING_ENABLED"
+	PROXY_PRUNING_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY = "PROXY_PRUNING_BACKEND_HOST_URL_MAP"
 	PROXY_SERVICE_PORT_ENVIRONMENT_KEY                 = "PROXY_SERVICE_PORT"
 	DATABASE_NAME_ENVIRONMENT_KEY                      = "DATABASE_NAME"
 	DATABASE_ENDPOINT_URL_ENVIRONMENT_KEY              = "DATABASE_ENDPOINT_URL"
@@ -81,6 +86,8 @@ const (
 	DEFAULT_DATABASE_WRITE_TIMEOUT_SECONDS                      = 10
 )
 
+var ErrEmptyHostMap = errors.New("backend host url map is empty")
+
 // EnvOrDefault fetches an environment variable value, or if not set returns the fallback value
 func EnvOrDefault(key string, fallback string) string {
 	if val, ok := os.LookupEnv(key); ok {
@@ -141,8 +148,9 @@ func ParseRawProxyBackendHostURLMap(raw string) (map[string]url.URL, error) {
 
 	entries := strings.Split(raw, PROXY_BACKEND_HOST_URL_MAP_ENTRY_DELIMITER)
 
-	if len(entries) < 1 {
-		return hostURLMap, fmt.Errorf("found zero mappings delimited by %s in %s", PROXY_BACKEND_HOST_URL_MAP_ENTRY_DELIMITER, raw)
+	if raw == "" || len(entries) < 1 {
+		extraErr := fmt.Errorf("found zero mappings delimited by %s in %s", PROXY_BACKEND_HOST_URL_MAP_ENTRY_DELIMITER, raw)
+		return hostURLMap, errors.Join(ErrEmptyHostMap, extraErr)
 	}
 
 	for _, entry := range entries {
@@ -175,15 +183,20 @@ func ParseRawProxyBackendHostURLMap(raw string) (map[string]url.URL, error) {
 // function of the Config package before use
 func ReadConfig() Config {
 	rawProxyBackendHostURLMap := os.Getenv(PROXY_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY)
-	// best effort to pares, callers are responsible for validating
+	rawProxyPruningBackendHostURLMap := os.Getenv(PROXY_PRUNING_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY)
+	// best effort to parse, callers are responsible for validating
 	// before using any values read
 	parsedProxyBackendHostURLMap, _ := ParseRawProxyBackendHostURLMap(rawProxyBackendHostURLMap)
+	parsedProxyPruningBackendHostURLMap, _ := ParseRawProxyBackendHostURLMap(rawProxyPruningBackendHostURLMap)
 
 	return Config{
 		ProxyServicePort:                       os.Getenv(PROXY_SERVICE_PORT_ENVIRONMENT_KEY),
 		LogLevel:                               EnvOrDefault(LOG_LEVEL_ENVIRONMENT_KEY, DEFAULT_LOG_LEVEL),
 		ProxyBackendHostURLMapRaw:              rawProxyBackendHostURLMap,
 		ProxyBackendHostURLMapParsed:           parsedProxyBackendHostURLMap,
+		EnableHeightBasedRouting:               EnvOrDefaultBool(PROXY_HEIGHT_BASED_ROUTING_ENABLED_KEY, false),
+		ProxyPruningBackendHostURLMapRaw:       rawProxyPruningBackendHostURLMap,
+		ProxyPruningBackendHostURLMap:          parsedProxyPruningBackendHostURLMap,
 		DatabaseName:                           os.Getenv(DATABASE_NAME_ENVIRONMENT_KEY),
 		DatabaseEndpointURL:                    os.Getenv(DATABASE_ENDPOINT_URL_ENVIRONMENT_KEY),
 		DatabaseUserName:                       os.Getenv(DATABASE_USERNAME_ENVIRONMENT_KEY),

config/config_test.go

@@ -9,9 +9,11 @@ import (
 )
 
 var (
-	proxyServicePort              = "7777"
-	randomEnvironmentVariableKey  = "TEST_KAVA_RANDOM_VALUE"
-	proxyServiceBackendHostURLMap = os.Getenv("TEST_PROXY_BACKEND_HOST_URL_MAP")
+	proxyServicePort                     = "7777"
+	randomEnvironmentVariableKey         = "TEST_KAVA_RANDOM_VALUE"
+	proxyServiceBackendHostURLMap        = os.Getenv("TEST_PROXY_BACKEND_HOST_URL_MAP")
+	proxyServiceHeightBasedRouting       = os.Getenv("TEST_PROXY_HEIGHT_BASED_ROUTING_ENABLED")
+	proxyServicePruningBackendHostURLMap = os.Getenv("TEST_PROXY_PRUNING_BACKEND_HOST_URL_MAP")
 )
 
 func TestUnitTestEnvODefaultReturnsDefaultIfEnvironmentVariableNotSet(t *testing.T) {
@@ -46,8 +48,15 @@ func TestUnitTestReadConfigReturnsConfigWithValuesFromEnv(t *testing.T) {
 	assert.Equal(t, proxyServicePort, readConfig.ProxyServicePort)
 }
 
+func TestUnitTestParseHostMapReturnsErrEmptyHostMapWhenEmpty(t *testing.T) {
+	_, err := config.ParseRawProxyBackendHostURLMap("")
+	assert.ErrorIs(t, err, config.ErrEmptyHostMap)
+}
+
 func setDefaultEnv() {
 	os.Setenv(config.PROXY_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY, proxyServiceBackendHostURLMap)
+	os.Setenv(config.PROXY_HEIGHT_BASED_ROUTING_ENABLED_KEY, proxyServiceHeightBasedRouting)
+	os.Setenv(config.PROXY_PRUNING_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY, proxyServicePruningBackendHostURLMap)
 	os.Setenv(config.PROXY_SERVICE_PORT_ENVIRONMENT_KEY, proxyServicePort)
 	os.Setenv(config.LOG_LEVEL_ENVIRONMENT_KEY, config.DEFAULT_LOG_LEVEL)
 }