From 4c4b5c63cc73161488a680b0facf0413544ddd46 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 6 Oct 2023 13:06:19 -0400 Subject: [PATCH 01/29] Basic implementation of Caching Middleware --- .env | 3 + .gitignore | 3 +- architecture/CACHING.md | 73 ++++++ clients/cache/cache.go | 16 ++ clients/cache/inmemory.go | 101 ++++++++ clients/cache/redis.go | 114 +++++++++ config/config.go | 13 + decode/evm_rpc.go | 19 +- go.mod | 3 + go.sum | 7 + main_test.go | 190 ++++++++++++++- service/cachemdw/cache.go | 125 ++++++++++ service/cachemdw/cache_test.go | 132 ++++++++++ service/cachemdw/caching_middleware.go | 54 ++++ service/cachemdw/doc.go | 10 + service/cachemdw/is_cached_middleware.go | 77 ++++++ service/cachemdw/keys.go | 67 +++++ service/cachemdw/keys_test.go | 71 ++++++ service/cachemdw/middleware_test.go | 110 +++++++++ service/cachemdw/testdata_test.go | 298 +++++++++++++++++++++++ service/handlers.go | 13 +- service/middleware.go | 48 +++- service/service.go | 84 +++++-- 23 files changed, 1601 insertions(+), 30 deletions(-) create mode 100644 architecture/CACHING.md create mode 100644 clients/cache/cache.go create mode 100644 clients/cache/inmemory.go create mode 100644 clients/cache/redis.go create mode 100644 service/cachemdw/cache.go create mode 100644 service/cachemdw/cache_test.go create mode 100644 service/cachemdw/caching_middleware.go create mode 100644 service/cachemdw/doc.go create mode 100644 service/cachemdw/is_cached_middleware.go create mode 100644 service/cachemdw/keys.go create mode 100644 service/cachemdw/keys_test.go create mode 100644 service/cachemdw/middleware_test.go create mode 100644 service/cachemdw/testdata_test.go diff --git a/.env b/.env index 5197d89..a54ad67 100644 --- a/.env +++ b/.env @@ -89,6 +89,9 @@ METRIC_PARTITIONING_ROUTINE_DELAY_FIRST_RUN_SECONDS=10 METRIC_PARTITIONINING_PREFILL_PERIOD_DAYS=7 # Used by `ready` script to ensure metric partitions have been created. MINIMUM_REQUIRED_PARTITIONS=30 +REDIS_ENDPOINT_URL=redis:6379 +REDIS_PASSWORD= +CHAIN_ID=local-chain ##### Database Config POSTGRES_PASSWORD=password diff --git a/.gitignore b/.gitignore index bb30b67..83ba227 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,8 @@ kava-proxy-service cover.html # Dependency directories (remove the comment below to include it) -# vendor/ +vendor/ # ignore editor files .vscode/ +.idea/ \ No newline at end of file diff --git a/architecture/CACHING.md b/architecture/CACHING.md new file mode 100644 index 0000000..218e762 --- /dev/null +++ b/architecture/CACHING.md @@ -0,0 +1,73 @@ +## Caching Middleware Architecture + +Package `cachemdw` is responsible for caching EVM requests and provides corresponding middleware + +package can work with any underlying storage which implements simple `cache.Cache` interface + +package provides two different middlewares: +- `IsCachedMiddleware` (should be run before proxy middleware) +- `CachingMiddleware` (should be run after proxy middleware) + +`IsCachedMiddleware` is responsible for setting response in the context if it's in the cache + +`CachingMiddleware` is responsible for caching response by taking a value from context (should be set by `ProxyMiddleware`) and setting in the cache + +## CachingMiddleware + +`CachingMiddleware` returns kava-proxy-service compatible middleware which works in the following way: +- tries to get decoded request from context (previous middleware should set it) +- checks few conditions: + - if request isn't already cached + - if request is cacheable + - if response is present in context +- if all above is true - caches the response +- calls next middleware + +## IsCachedMiddleware + +`IsCachedMiddleware` returns kava-proxy-service compatible middleware which works in the following way: +- tries to get decoded request from context (previous middleware should set it) +- tries to get response from the cache + - if present sets cached response in context, marks as cached in context and forwards to next middleware + - if not present marks as uncached in context and forwards to next middleware +- next middleware should check whether request was cached and act accordingly: + +## Cache Invalidation + +### Keys Structure + +Keys have such format: + +`query:::` + +For example: + +`query:local-chain:eth_getBlockByNumber:0x72806e50da4f1c824b9d5a74ce9d76ac4db72e4da049802d1d6f2de3fda73e10` + +### Invalidation for specific method + +If you want to invalidate cache for specific method you may run such command: + +`redis-cli KEYS "query:::*" | xargs redis-cli DEL` + +For example: + +`redis-cli KEYS "query:local-chain:eth_getBlockByNumber:*" | xargs redis-cli DEL` + +### Invalidation for all methods + +If you want to invalidate cache for all methods you may run such command: + +`redis-cli KEYS "query::*" | xargs redis-cli DEL` + +For example: + +`redis-cli KEYS "query:local-chain:*" | xargs redis-cli DEL` + +## Architecture Diagrams + +### Serve request from the cache (avoiding call to actual backend) +![image](https://github.com/Kava-Labs/kava-proxy-service/assets/37836031/1bd8cb8e-6a9e-45a6-b698-3f99eaab2aa2) + +### Serve request from the backend and then cache the response +![image](https://github.com/Kava-Labs/kava-proxy-service/assets/37836031/b0eb5cb9-51da-43f9-bb7d-b94bf482f366) diff --git a/clients/cache/cache.go b/clients/cache/cache.go new file mode 100644 index 0000000..cc6f9cb --- /dev/null +++ b/clients/cache/cache.go @@ -0,0 +1,16 @@ +package cache + +import ( + "context" + "errors" + "time" +) + +var ErrNotFound = errors.New("value not found in the cache") + +type Cache interface { + Set(ctx context.Context, key string, data []byte, expiration time.Duration) error + Get(ctx context.Context, key string) ([]byte, error) + Delete(ctx context.Context, key string) error + Healthcheck(ctx context.Context) error +} diff --git a/clients/cache/inmemory.go b/clients/cache/inmemory.go new file mode 100644 index 0000000..9293af2 --- /dev/null +++ b/clients/cache/inmemory.go @@ -0,0 +1,101 @@ +package cache + +import ( + "context" + "sync" + "time" +) + +// InMemoryCache is an in-memory implementation of the Cache interface. +type InMemoryCache struct { + data map[string]cacheItem + mutex sync.RWMutex +} + +// Ensure InMemoryCache implements the Cache interface. +var _ Cache = (*InMemoryCache)(nil) + +// cacheItem represents an item stored in the cache. +type cacheItem struct { + data []byte + expiration time.Time +} + +// NewInMemoryCache creates a new instance of InMemoryCache. +func NewInMemoryCache() *InMemoryCache { + return &InMemoryCache{ + data: make(map[string]cacheItem), + } +} + +// Set sets the value of a key in the cache. +func (c *InMemoryCache) Set( + ctx context.Context, + key string, + data []byte, + expiration time.Duration, +) error { + c.mutex.Lock() + defer c.mutex.Unlock() + + expiry := time.Now().Add(expiration) + + if expiration == 0 { + // 100 years in the future to prevent expiry + expiry = time.Now().AddDate(100, 0, 0) + } + + c.data[key] = cacheItem{ + data: data, + expiration: expiry, + } + + return nil +} + +// Get retrieves the value of a key from the cache. +func (c *InMemoryCache) Get(ctx context.Context, key string) ([]byte, error) { + c.mutex.RLock() + defer c.mutex.RUnlock() + + item, ok := c.data[key] + if !ok || time.Now().After(item.expiration) { + // Not a real ttl but just replicates it for fetching + delete(c.data, key) + + return nil, ErrNotFound + } + + return item.data, nil +} + +// GetAll returns all the non-expired data in the cache. +func (c *InMemoryCache) GetAll(ctx context.Context) map[string][]byte { + c.mutex.RLock() + defer c.mutex.RUnlock() + + result := make(map[string][]byte) + + for key, item := range c.data { + if time.Now().After(item.expiration) { + delete(c.data, key) + } else { + result[key] = item.data + } + } + + return result +} + +// Delete removes a key from the cache. +func (c *InMemoryCache) Delete(ctx context.Context, key string) error { + c.mutex.Lock() + defer c.mutex.Unlock() + + delete(c.data, key) + return nil +} + +func (c *InMemoryCache) Healthcheck(ctx context.Context) error { + return nil +} diff --git a/clients/cache/redis.go b/clients/cache/redis.go new file mode 100644 index 0000000..f5bc604 --- /dev/null +++ b/clients/cache/redis.go @@ -0,0 +1,114 @@ +package cache + +import ( + "context" + "fmt" + "time" + + "github.com/kava-labs/kava-proxy-service/logging" + "github.com/redis/go-redis/v9" +) + +type RedisConfig struct { + Address string + Password string + DB int +} + +// RedisCache is an implementation of Cache that uses Redis as the caching backend. +type RedisCache struct { + client *redis.Client + *logging.ServiceLogger +} + +var _ Cache = (*RedisCache)(nil) + +func NewRedisCache( + cfg *RedisConfig, + logger *logging.ServiceLogger, +) (*RedisCache, error) { + client := redis.NewClient(&redis.Options{ + Addr: cfg.Address, + Password: cfg.Password, + DB: cfg.DB, + }) + + return &RedisCache{ + client: client, + ServiceLogger: logger, + }, nil +} + +// Set sets the value for the given key in the cache with the given expiration. +func (rc *RedisCache) Set( + ctx context.Context, + key string, + value []byte, + expiration time.Duration, +) error { + rc.Logger.Trace(). + Str("key", key). + Str("value", string(value)). + Dur("expiration", expiration). + Msg("setting value in redis") + + return rc.client.Set(ctx, key, value, expiration).Err() +} + +// Get gets the value for the given key in the cache. +func (rc *RedisCache) Get( + ctx context.Context, + key string, +) ([]byte, error) { + rc.Logger.Trace(). + Str("key", key). + Msg("getting value from redis") + + val, err := rc.client.Get(ctx, key).Bytes() + if err == redis.Nil { + rc.Logger.Trace(). + Str("key", key). + Msgf("value not found in redis") + return nil, ErrNotFound + } + if err != nil { + rc.Logger.Error(). + Str("key", key). + Err(err). + Msg("error during getting value from redis") + return nil, err + } + + rc.Logger.Trace(). + Str("key", key). + Str("value", string(val)). + Msg("successfully got value from redis") + + return val, nil +} + +// Delete deletes the value for the given key in the cache. +func (rc *RedisCache) Delete(ctx context.Context, key string) error { + rc.Logger.Trace(). + Str("key", key). + Msg("deleting value from redis") + + return rc.client.Del(ctx, key).Err() +} + +func (rc *RedisCache) Healthcheck(ctx context.Context) error { + rc.Logger.Trace().Msg("redis healthcheck was called") + + // Check if we can connect to Redis + _, err := rc.client.Ping(ctx).Result() + if err != nil { + rc.Logger.Error(). + Err(err). + Msg("can't ping redis") + return fmt.Errorf("error connecting to Redis: %v", err) + } + + rc.Logger.Trace().Msg("redis healthcheck was successful") + + return nil +} diff --git a/config/config.go b/config/config.go index 06e2d73..4891135 100644 --- a/config/config.go +++ b/config/config.go @@ -37,6 +37,10 @@ type Config struct { MetricPartitioningRoutineInterval time.Duration MetricPartitioningRoutineDelayFirstRun time.Duration MetricPartitioningPrefillPeriodDays int + RedisEndpointURL string + RedisPassword string + CacheTTL time.Duration + ChainID string } const ( @@ -79,6 +83,11 @@ const ( DEFAULT_DATABASE_READ_TIMEOUT_SECONDS = 60 DATABASE_WRITE_TIMEOUT_SECONDS_ENVIRONMENT_KEY = "DATABASE_WRITE_TIMEOUT_SECONDS" DEFAULT_DATABASE_WRITE_TIMEOUT_SECONDS = 10 + REDIS_ENDPOINT_URL_ENVIRONMENT_KEY = "REDIS_ENDPOINT_URL" + REDIS_PASSWORD_ENVIRONMENT_KEY = "REDIS_PASSWORD" + CACHE_TTL_ENVIRONMENT_KEY = "CACHE_TTL" + DEFAULT_CACHE_TTL_SECONDS = 600 + CHAIN_ID_ENVIRONMENT_KEY = "CHAIN_ID" ) // EnvOrDefault fetches an environment variable value, or if not set returns the fallback value @@ -204,5 +213,9 @@ func ReadConfig() Config { MetricPartitioningRoutineInterval: time.Duration(time.Duration(EnvOrDefaultInt(METRIC_PARTITIONING_ROUTINE_INTERVAL_SECONDS_ENVIRONMENT_KEY, DEFAULT_METRIC_PARTITIONING_ROUTINE_INTERVAL_SECONDS)) * time.Second), MetricPartitioningRoutineDelayFirstRun: time.Duration(time.Duration(EnvOrDefaultInt(METRIC_PARTITIONING_ROUTINE_DELAY_FIRST_RUN_SECONDS_ENVIRONMENT_KEY, DEFAULT_METRIC_PARTITIONING_ROUTINE_DELAY_FIRST_RUN_SECONDS)) * time.Second), MetricPartitioningPrefillPeriodDays: EnvOrDefaultInt(METRIC_PARTITIONING_PREFILL_PERIOD_DAYS_ENVIRONMENT_KEY, DEFAULT_METRIC_PARTITIONING_PREFILL_PERIOD_DAYS), + RedisEndpointURL: os.Getenv(REDIS_ENDPOINT_URL_ENVIRONMENT_KEY), + RedisPassword: os.Getenv(REDIS_PASSWORD_ENVIRONMENT_KEY), + CacheTTL: time.Duration(EnvOrDefaultInt(CACHE_TTL_ENVIRONMENT_KEY, DEFAULT_CACHE_TTL_SECONDS)) * time.Second, + ChainID: os.Getenv(CHAIN_ID_ENVIRONMENT_KEY), } } diff --git a/decode/evm_rpc.go b/decode/evm_rpc.go index c773792..4ead817 100644 --- a/decode/evm_rpc.go +++ b/decode/evm_rpc.go @@ -6,12 +6,17 @@ import ( "errors" "fmt" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethclient" - cosmosmath "cosmossdk.io/math" + "github.com/ethereum/go-ethereum/common" + ethctypes "github.com/ethereum/go-ethereum/core/types" ) +// EVMBlockGetter defines an interface which can be implemented by any client capable of getting ethereum block by hash +type EVMBlockGetter interface { + // BlockByHash returns ethereum block by hash + BlockByHash(ctx context.Context, hash common.Hash) (*ethctypes.Block, error) +} + // Errors that might result from decoding parts or the whole of // an EVM RPC request var ( @@ -138,7 +143,7 @@ func DecodeEVMRPCRequest(body []byte) (*EVMRPCRequestEnvelope, error) { // - the request is a valid evm rpc request // - the method for the request supports specifying a block number // - the provided block number is a valid tag or number -func (r *EVMRPCRequestEnvelope) ExtractBlockNumberFromEVMRPCRequest(ctx context.Context, evmClient *ethclient.Client) (int64, error) { +func (r *EVMRPCRequestEnvelope) ExtractBlockNumberFromEVMRPCRequest(ctx context.Context, blockGetter EVMBlockGetter) (int64, error) { // only attempt to extract block number from a valid ethereum api request if r.Method == "" { return 0, ErrInvalidEthAPIRequest @@ -173,12 +178,12 @@ func (r *EVMRPCRequestEnvelope) ExtractBlockNumberFromEVMRPCRequest(ctx context. return parseBlockNumberFromParams(r.Method, r.Params) } - return lookupBlockNumberFromHashParam(ctx, evmClient, r.Method, r.Params) + return lookupBlockNumberFromHashParam(ctx, blockGetter, r.Method, r.Params) } // Generic method to lookup the block number // based on the hash value in a set of params -func lookupBlockNumberFromHashParam(ctx context.Context, evmClient *ethclient.Client, methodName string, params []interface{}) (int64, error) { +func lookupBlockNumberFromHashParam(ctx context.Context, blockGetter EVMBlockGetter, methodName string, params []interface{}) (int64, error) { paramIndex, exists := MethodNameToBlockHashParamIndex[methodName] if !exists { @@ -191,7 +196,7 @@ func lookupBlockNumberFromHashParam(ctx context.Context, evmClient *ethclient.Cl return 0, fmt.Errorf(fmt.Sprintf("error decoding block hash param from params %+v at index %d", params, paramIndex)) } - block, err := evmClient.BlockByHash(ctx, common.HexToHash(blockHash)) + block, err := blockGetter.BlockByHash(ctx, common.HexToHash(blockHash)) if err != nil { return 0, err diff --git a/go.mod b/go.mod index 27a36ba..8bc47b3 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( cosmossdk.io/math v1.0.0 github.com/ethereum/go-ethereum v1.11.2 github.com/google/uuid v1.3.0 + github.com/redis/go-redis/v9 v9.2.1 github.com/rs/zerolog v1.29.0 github.com/stretchr/testify v1.8.2 github.com/uptrace/bun v1.1.12 @@ -18,9 +19,11 @@ require ( require ( github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fatih/color v1.14.1 // indirect github.com/go-ole/go-ole v1.2.1 // indirect github.com/go-stack/stack v1.8.1 // indirect diff --git a/go.sum b/go.sum index a82a2c9..55e2f8e 100644 --- a/go.sum +++ b/go.sum @@ -5,10 +5,13 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIO github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk= @@ -24,6 +27,8 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/ethereum/go-ethereum v1.11.2 h1:z/luyejbevDCAMUUiu0rc80dxJxOnpoG58k5o0tSawc= github.com/ethereum/go-ethereum v1.11.2/go.mod h1:DuefStAgaxoaYGLR0FueVcVbehmn5n9QUcVrMCuOvuc= @@ -78,6 +83,8 @@ github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvq github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg= +github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= diff --git a/main_test.go b/main_test.go index 3bd2cff..b729d24 100644 --- a/main_test.go +++ b/main_test.go @@ -1,7 +1,14 @@ package main_test import ( + "bytes" "context" + "encoding/json" + "errors" + "fmt" + "io" + "math/big" + "net/http" "os" "testing" "time" @@ -9,11 +16,14 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethclient" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/kava-labs/kava-proxy-service/clients/database" "github.com/kava-labs/kava-proxy-service/decode" "github.com/kava-labs/kava-proxy-service/logging" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" + "github.com/kava-labs/kava-proxy-service/service/cachemdw" ) const ( @@ -50,6 +60,9 @@ var ( Logger: &testServiceLogger, RunDatabaseMigrations: false, } + + redisHostPort = os.Getenv("REDIS_HOST_PORT") + redisPassword = os.Getenv("REDIS_PASSWORD") ) func TestE2ETestProxyReturnsNonZeroLatestBlockHeader(t *testing.T) { @@ -469,3 +482,176 @@ func TestE2ETestProxyTracksBlockNumberForMethodsWithBlockHashParam(t *testing.T) assert.Equal(t, *requestMetricDuringRequestWindow.BlockNumber, requestBlockNumber) } } + +func TestE2ETestProxyCachesMethodsWithBlockNumberParam(t *testing.T) { + testRandomAddressHex := "0x6767114FFAA17C6439D7AEA480738B982CE63A02" + testAddress := common.HexToAddress(testRandomAddressHex) + + // create api and database clients + client, err := ethclient.Dial(proxyServiceURL) + if err != nil { + t.Fatal(err) + } + + redisClient := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("localhost:%v", redisHostPort), + Password: redisPassword, + DB: 0, + }) + cleanUpRedis(t, redisClient) + expectKeysNum(t, redisClient, 0) + + for _, tc := range []struct { + desc string + method string + params []interface{} + keysNum int + }{ + { + desc: "test case #1", + method: "eth_getTransactionCount", + params: []interface{}{testAddress, "0x1"}, + keysNum: 1, + }, + { + desc: "test case #2", + method: "eth_getBlockByNumber", + params: []interface{}{"0x1", true}, + keysNum: 2, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + // test cache MISS and cache HIT scenarios for specified method + // check corresponding values in cachemdw.CacheMissHeaderValue HTTP header + // check that cached and non-cached responses are equal + + // eth_getBlockByNumber - cache MISS + resp1 := mkJsonRpcRequest(t, proxyServiceURL, tc.method, tc.params) + require.Equal(t, cachemdw.CacheMissHeaderValue, resp1.Header[cachemdw.CacheHeaderKey][0]) + body1, err := io.ReadAll(resp1.Body) + require.NoError(t, err) + err = checkJsonRpcErr(body1) + require.NoError(t, err) + expectKeysNum(t, redisClient, tc.keysNum) + + // eth_getBlockByNumber - cache HIT + resp2 := mkJsonRpcRequest(t, proxyServiceURL, tc.method, tc.params) + require.Equal(t, cachemdw.CacheHitHeaderValue, resp2.Header[cachemdw.CacheHeaderKey][0]) + body2, err := io.ReadAll(resp2.Body) + require.NoError(t, err) + err = checkJsonRpcErr(body2) + require.NoError(t, err) + expectKeysNum(t, redisClient, tc.keysNum) + + require.JSONEq(t, string(body1), string(body2), "blocks should be the same") + }) + } + + // test cache MISS and cache HIT scenarios for eth_getTransactionCount method + // check that cached and non-cached responses are equal + { + // eth_getTransactionCount - cache MISS + bal1, err := client.NonceAt(testContext, testAddress, big.NewInt(2)) + require.NoError(t, err) + expectKeysNum(t, redisClient, 3) + + // eth_getTransactionCount - cache HIT + bal2, err := client.NonceAt(testContext, testAddress, big.NewInt(2)) + require.NoError(t, err) + expectKeysNum(t, redisClient, 3) + + require.Equal(t, bal1, bal2, "balances should be the same") + } + + // test cache MISS and cache HIT scenarios for eth_getBlockByNumber method + // check that cached and non-cached responses are equal + { + // eth_getBlockByNumber - cache MISS + block1, err := client.BlockByNumber(testContext, big.NewInt(2)) + require.NoError(t, err) + expectKeysNum(t, redisClient, 4) + + // eth_getBlockByNumber - cache HIT + block2, err := client.BlockByNumber(testContext, big.NewInt(2)) + require.NoError(t, err) + expectKeysNum(t, redisClient, 4) + + require.Equal(t, block1, block2, "blocks should be the same") + } + + cleanUpRedis(t, redisClient) +} + +func expectKeysNum(t *testing.T, redisClient *redis.Client, keysNum int) { + keys, err := redisClient.Keys(context.Background(), "*").Result() + require.NoError(t, err) + + require.Equal(t, keysNum, len(keys)) +} + +func cleanUpRedis(t *testing.T, redisClient *redis.Client) { + keys, err := redisClient.Keys(context.Background(), "*").Result() + require.NoError(t, err) + + for _, key := range keys { + fmt.Printf("key: %v\n", key) + } + + if len(keys) != 0 { + _, err = redisClient.Del(context.Background(), keys...).Result() + require.NoError(t, err) + } +} + +func mkJsonRpcRequest(t *testing.T, proxyServiceURL, method string, params []interface{}) *http.Response { + req := newJsonRpcRequest(method, params) + reqInJSON, err := json.Marshal(req) + require.NoError(t, err) + reqReader := bytes.NewBuffer(reqInJSON) + resp, err := http.Post(proxyServiceURL, "application/json", reqReader) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + + return resp +} + +type jsonRpcRequest struct { + JsonRpc string `json:"jsonrpc"` + Method string `json:"method"` + Params []interface{} `json:"params"` + Id int `json:"id"` +} + +func newJsonRpcRequest(method string, params []interface{}) *jsonRpcRequest { + return &jsonRpcRequest{ + JsonRpc: "2.0", + Method: method, + Params: params, + Id: 1, + } +} + +type jsonRpcResponse struct { + Jsonrpc string `json:"jsonrpc"` + Id int `json:"id"` + Result interface{} `json:"result"` + Error string `json:"error"` +} + +func checkJsonRpcErr(body []byte) error { + var resp jsonRpcResponse + err := json.Unmarshal(body, &resp) + if err != nil { + return err + } + + if resp.Error != "" { + return errors.New(resp.Error) + } + + if resp.Result == "" { + return errors.New("result is empty") + } + + return nil +} diff --git a/service/cachemdw/cache.go b/service/cachemdw/cache.go new file mode 100644 index 0000000..ef25f25 --- /dev/null +++ b/service/cachemdw/cache.go @@ -0,0 +1,125 @@ +package cachemdw + +import ( + "context" + "errors" + "time" + + "github.com/kava-labs/kava-proxy-service/clients/cache" + "github.com/kava-labs/kava-proxy-service/decode" + "github.com/kava-labs/kava-proxy-service/logging" +) + +// ServiceCache is responsible for caching EVM requests and provides corresponding middleware +// ServiceCache can work with any underlying storage which implements simple cache.Cache interface +type ServiceCache struct { + cacheClient cache.Cache + blockGetter decode.EVMBlockGetter + cacheTTL time.Duration + decodedRequestContextKey any + // chainID is used as prefix for any key in the cache + chainID string + + *logging.ServiceLogger +} + +func NewServiceCache( + cacheClient cache.Cache, + blockGetter decode.EVMBlockGetter, + cacheTTL time.Duration, + decodedRequestContextKey any, + chainID string, + logger *logging.ServiceLogger, +) *ServiceCache { + return &ServiceCache{ + cacheClient: cacheClient, + blockGetter: blockGetter, + cacheTTL: cacheTTL, + decodedRequestContextKey: decodedRequestContextKey, + chainID: chainID, + ServiceLogger: logger, + } +} + +// IsCacheable checks if EVM request is cacheable. +// In current implementation we consider request is cacheable if it has specific block height +func IsCacheable( + ctx context.Context, + blockGetter decode.EVMBlockGetter, + logger *logging.ServiceLogger, + req *decode.EVMRPCRequestEnvelope, +) bool { + blockNumber, err := req.ExtractBlockNumberFromEVMRPCRequest(ctx, blockGetter) + if err != nil { + logger.Logger.Error(). + Err(err). + Msg("can't extract block number from EVM RPC request") + return false + } + + if blockNumber <= 0 { + return false + } + + return true +} + +// GetCachedQueryResponse calculates cache key for request and then tries to get it from cache. +func (c *ServiceCache) GetCachedQueryResponse( + ctx context.Context, + req *decode.EVMRPCRequestEnvelope, +) ([]byte, error) { + key, err := GetQueryKey(c.chainID, req) + if err != nil { + return nil, err + } + + value, err := c.cacheClient.Get(ctx, key) + if err != nil { + return nil, err + } + + return value, nil +} + +// CacheQueryResponse calculates cache key for request and then saves response to the cache. +func (c *ServiceCache) CacheQueryResponse( + ctx context.Context, + req *decode.EVMRPCRequestEnvelope, + chainID string, + response []byte, +) error { + if !IsCacheable(ctx, c.blockGetter, c.ServiceLogger, req) { + return errors.New("query isn't cacheable") + } + + key, err := GetQueryKey(chainID, req) + if err != nil { + return err + } + + return c.cacheClient.Set(ctx, key, response, c.cacheTTL) +} + +func (c *ServiceCache) ValidateAndCacheQueryResponse( + ctx context.Context, + req *decode.EVMRPCRequestEnvelope, + response []byte, +) error { + // TODO(yevhenii): add validation + + if err := c.CacheQueryResponse( + ctx, + req, + c.chainID, + response, + ); err != nil { + return err + } + + return nil +} + +func (c *ServiceCache) Healthcheck(ctx context.Context) error { + return c.cacheClient.Healthcheck(ctx) +} diff --git a/service/cachemdw/cache_test.go b/service/cachemdw/cache_test.go new file mode 100644 index 0000000..4675145 --- /dev/null +++ b/service/cachemdw/cache_test.go @@ -0,0 +1,132 @@ +package cachemdw_test + +import ( + "context" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + ethctypes "github.com/ethereum/go-ethereum/core/types" + "github.com/stretchr/testify/require" + + "github.com/kava-labs/kava-proxy-service/clients/cache" + "github.com/kava-labs/kava-proxy-service/decode" + "github.com/kava-labs/kava-proxy-service/logging" + "github.com/kava-labs/kava-proxy-service/service" + "github.com/kava-labs/kava-proxy-service/service/cachemdw" +) + +const ( + defaultChainIDString = "1" + defaultHost = "api.kava.io" + defaultBlockNumber = "42" +) + +var ( + defaultChainID = big.NewInt(1) + defaultQueryResp = []byte("resp") +) + +type MockEVMBlockGetter struct{} + +func NewMockEVMBlockGetter() *MockEVMBlockGetter { + return &MockEVMBlockGetter{} +} + +var _ decode.EVMBlockGetter = (*MockEVMBlockGetter)(nil) + +func (c *MockEVMBlockGetter) BlockByHash(ctx context.Context, hash common.Hash) (*ethctypes.Block, error) { + panic("not implemented") +} + +func (c *MockEVMBlockGetter) ChainID(ctx context.Context) (*big.Int, error) { + return defaultChainID, nil +} + +func TestUnitTestIsCacheable(t *testing.T) { + logger, err := logging.New("TRACE") + require.NoError(t, err) + + blockGetter := NewMockEVMBlockGetter() + ctxb := context.Background() + + for _, tc := range []struct { + desc string + req *decode.EVMRPCRequestEnvelope + cacheable bool + }{ + { + desc: "test case #1", + req: mkEVMRPCRequestEnvelope(defaultBlockNumber), + cacheable: true, + }, + { + desc: "test case #2", + req: mkEVMRPCRequestEnvelope("0"), + cacheable: false, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + cacheable := cachemdw.IsCacheable(ctxb, blockGetter, &logger, tc.req) + require.Equal(t, tc.cacheable, cacheable) + }) + } +} + +func TestUnitTestCacheQueryResponse(t *testing.T) { + logger, err := logging.New("TRACE") + require.NoError(t, err) + + inMemoryCache := cache.NewInMemoryCache() + blockGetter := NewMockEVMBlockGetter() + cacheTTL := time.Hour + ctxb := context.Background() + + serviceCache := cachemdw.NewServiceCache(inMemoryCache, blockGetter, cacheTTL, service.DecodedRequestContextKey, defaultChainIDString, &logger) + + req := mkEVMRPCRequestEnvelope(defaultBlockNumber) + resp, err := serviceCache.GetCachedQueryResponse(ctxb, req) + require.Equal(t, cache.ErrNotFound, err) + require.Empty(t, resp) + + err = serviceCache.CacheQueryResponse(ctxb, req, defaultChainIDString, defaultQueryResp) + require.NoError(t, err) + + resp, err = serviceCache.GetCachedQueryResponse(ctxb, req) + require.NoError(t, err) + require.Equal(t, defaultQueryResp, resp) +} + +func TestUnitTestValidateAndCacheQueryResponse(t *testing.T) { + logger, err := logging.New("TRACE") + require.NoError(t, err) + + inMemoryCache := cache.NewInMemoryCache() + blockGetter := NewMockEVMBlockGetter() + cacheTTL := time.Hour + ctxb := context.Background() + + serviceCache := cachemdw.NewServiceCache(inMemoryCache, blockGetter, cacheTTL, service.DecodedRequestContextKey, defaultChainIDString, &logger) + + req := mkEVMRPCRequestEnvelope(defaultBlockNumber) + resp, err := serviceCache.GetCachedQueryResponse(ctxb, req) + require.Equal(t, cache.ErrNotFound, err) + require.Empty(t, resp) + + err = serviceCache.ValidateAndCacheQueryResponse(ctxb, req, defaultQueryResp) + require.NoError(t, err) + + resp, err = serviceCache.GetCachedQueryResponse(ctxb, req) + require.NoError(t, err) + require.Equal(t, defaultQueryResp, resp) +} + +func mkEVMRPCRequestEnvelope(blockNumber string) *decode.EVMRPCRequestEnvelope { + return &decode.EVMRPCRequestEnvelope{ + JSONRPCVersion: "2.0", + ID: 1, + Method: "eth_getBalance", + Params: []interface{}{"0x1234", blockNumber}, + } +} diff --git a/service/cachemdw/caching_middleware.go b/service/cachemdw/caching_middleware.go new file mode 100644 index 0000000..25f1ae2 --- /dev/null +++ b/service/cachemdw/caching_middleware.go @@ -0,0 +1,54 @@ +package cachemdw + +import ( + "net/http" + + "github.com/kava-labs/kava-proxy-service/decode" +) + +// CachingMiddleware returns kava-proxy-service compatible middleware which works in the following way: +// - tries to get decoded request from context (previous middleware should set it) +// - checks few conditions: +// - if request isn't already cached +// - if request is cacheable +// - if response is present in context +// +// - if all above is true - caches the response +// - calls next middleware +func (c *ServiceCache) CachingMiddleware( + next http.Handler, +) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // if we can't get decoded request then forward to next middleware + req := r.Context().Value(c.decodedRequestContextKey) + decodedReq, ok := (req).(*decode.EVMRPCRequestEnvelope) + if !ok { + c.Logger.Error(). + Str("method", r.Method). + Str("url", r.URL.String()). + Str("host", r.Host). + Msg("can't cast request to *EVMRPCRequestEnvelope type") + + next.ServeHTTP(w, r) + return + } + + isCached := IsRequestCached(r.Context()) + cacheable := IsCacheable(r.Context(), c.blockGetter, c.ServiceLogger, decodedReq) + response := r.Context().Value(ResponseContextKey) + typedResponse, ok := response.([]byte) + + // if request isn't already cached, request is cacheable and response is present in context - cache the response + if !isCached && cacheable && ok { + if err := c.ValidateAndCacheQueryResponse( + r.Context(), + decodedReq, + typedResponse, + ); err != nil { + c.Logger.Error().Msgf("can't validate and cache response: %v", err) + } + } + + next.ServeHTTP(w, r) + } +} diff --git a/service/cachemdw/doc.go b/service/cachemdw/doc.go new file mode 100644 index 0000000..aafc683 --- /dev/null +++ b/service/cachemdw/doc.go @@ -0,0 +1,10 @@ +// Package cachemdw is responsible for caching EVM requests and provides corresponding middleware +// package can work with any underlying storage which implements simple cache.Cache interface +// +// package provides two different middlewares: +// - IsCachedMiddleware (should be run before proxy middleware) +// - CachingMiddleware (should be run after proxy middleware) +// +// IsCachedMiddleware is responsible for setting response in the context if it's in the cache +// CachingMiddleware is responsible for caching response by taking a value from context (should be set by proxy mdw) and setting in the cache +package cachemdw diff --git a/service/cachemdw/is_cached_middleware.go b/service/cachemdw/is_cached_middleware.go new file mode 100644 index 0000000..3cbc557 --- /dev/null +++ b/service/cachemdw/is_cached_middleware.go @@ -0,0 +1,77 @@ +package cachemdw + +import ( + "context" + "net/http" + + "github.com/kava-labs/kava-proxy-service/clients/cache" + "github.com/kava-labs/kava-proxy-service/decode" +) + +const ( + CachedContextKey = "X-KAVA-PROXY-CACHED" + ResponseContextKey = "X-KAVA-PROXY-RESPONSE" + + CacheHeaderKey = "X-Kava-Proxy-Cache-Status" + CacheHitHeaderValue = "HIT" + CacheMissHeaderValue = "MISS" +) + +// IsCachedMiddleware returns kava-proxy-service compatible middleware which works in the following way: +// - tries to get decoded request from context (previous middleware should set it) +// - tries to get response from the cache +// - if present sets cached response in context, marks as cached in context and forwards to next middleware +// - if not present marks as uncached in context and forwards to next middleware +// +// - next middleware should check whether request was cached and act accordingly: +func (c *ServiceCache) IsCachedMiddleware( + next http.Handler, +) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + uncachedContext := context.WithValue(r.Context(), CachedContextKey, false) + cachedContext := context.WithValue(r.Context(), CachedContextKey, true) + + // if we can't get decoded request then forward to next middleware + req := r.Context().Value(c.decodedRequestContextKey) + decodedReq, ok := (req).(*decode.EVMRPCRequestEnvelope) + if !ok { + c.Logger.Error(). + Str("method", r.Method). + Str("url", r.URL.String()). + Str("host", r.Host). + Msg("can't cast request to *EVMRPCRequestEnvelope type") + + next.ServeHTTP(w, r.WithContext(uncachedContext)) + return + } + + // Check if the request is cached: + // 1. if not cached or we encounter an error then mark as uncached and forward to next middleware + // 2. if cached then mark as cached, set cached response in context and forward to next middleware + cachedQueryResponse, err := c.GetCachedQueryResponse(r.Context(), decodedReq) + if err != nil && err != cache.ErrNotFound { + // log unexpected error + c.Logger.Error(). + Err(err). + Msg("error during getting response from cache") + } + if err != nil { + // 1. if not cached or we encounter an error then mark as uncached and forward to next middleware + next.ServeHTTP(w, r.WithContext(uncachedContext)) + return + } + + // 2. if cached then mark as cached, set cached response in context and forward to next middleware + responseContext := context.WithValue(cachedContext, ResponseContextKey, cachedQueryResponse) + next.ServeHTTP(w, r.WithContext(responseContext)) + } +} + +// IsRequestCached returns whether request was cached +// if returns true it means: +// - middleware marked that request was cached +// - value of cached response should be available in context via ResponseContextKey +func IsRequestCached(ctx context.Context) bool { + cached, ok := ctx.Value(CachedContextKey).(bool) + return ok && cached +} diff --git a/service/cachemdw/keys.go b/service/cachemdw/keys.go new file mode 100644 index 0000000..f3ab361 --- /dev/null +++ b/service/cachemdw/keys.go @@ -0,0 +1,67 @@ +package cachemdw + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/ethereum/go-ethereum/crypto" + + "github.com/kava-labs/kava-proxy-service/decode" +) + +type CacheItemType int + +const ( + CacheItemTypeQuery CacheItemType = iota + 1 +) + +func (t CacheItemType) String() string { + switch t { + case CacheItemTypeQuery: + return "query" + default: + return "unknown" + } +} + +func BuildCacheKey(cacheItemType CacheItemType, parts []string) string { + fullParts := append( + []string{ + cacheItemType.String(), + }, + parts..., + ) + + return strings.Join(fullParts, ":") +} + +// GetQueryKey calculates cache key for request +func GetQueryKey( + chainID string, + req *decode.EVMRPCRequestEnvelope, +) (string, error) { + if req == nil { + return "", fmt.Errorf("request shouldn't be nil") + } + + // TODO(yevhenii): use stable/sorted JSON serializer + serializedParams, err := json.Marshal(req.Params) + if err != nil { + return "", err + } + + data := make([]byte, 0) + data = append(data, []byte(req.Method)...) + data = append(data, serializedParams...) + + hashedReq := crypto.Keccak256Hash(data) + + parts := []string{ + chainID, + req.Method, + hashedReq.Hex(), + } + + return BuildCacheKey(CacheItemTypeQuery, parts), nil +} diff --git a/service/cachemdw/keys_test.go b/service/cachemdw/keys_test.go new file mode 100644 index 0000000..d67574f --- /dev/null +++ b/service/cachemdw/keys_test.go @@ -0,0 +1,71 @@ +package cachemdw_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kava-labs/kava-proxy-service/decode" + "github.com/kava-labs/kava-proxy-service/service/cachemdw" +) + +func TestUnitTestBuildCacheKey(t *testing.T) { + for _, tc := range []struct { + desc string + cacheItemType cachemdw.CacheItemType + parts []string + expectedCacheKey string + }{ + { + desc: "test case #1", + cacheItemType: cachemdw.CacheItemTypeQuery, + parts: []string{"1", "2", "3"}, + expectedCacheKey: "query:1:2:3", + }, + } { + t.Run(tc.desc, func(t *testing.T) { + cacheKey := cachemdw.BuildCacheKey(tc.cacheItemType, tc.parts) + require.Equal(t, tc.expectedCacheKey, cacheKey) + }) + } +} + +func TestUnitTestGetQueryKey(t *testing.T) { + for _, tc := range []struct { + desc string + chainID string + req *decode.EVMRPCRequestEnvelope + expectedCacheKey string + errMsg string + }{ + { + desc: "test case #1", + chainID: "chain1", + req: &decode.EVMRPCRequestEnvelope{ + JSONRPCVersion: "2.0", + ID: 1, + Method: "eth_getBlockByHash", + Params: []interface{}{"0x1234", true}, + }, + expectedCacheKey: "query:chain1:eth_getBlockByHash:0xb2b69f976d9aa41cd2065e2a2354254f6cba682a6fe2b3996571daa27ea4a6f4", + }, + { + desc: "test case #1", + chainID: "chain1", + req: nil, + errMsg: "request shouldn't be nil", + }, + } { + t.Run(tc.desc, func(t *testing.T) { + cacheKey, err := cachemdw.GetQueryKey(tc.chainID, tc.req) + if tc.errMsg == "" { + require.NoError(t, err) + require.Equal(t, tc.expectedCacheKey, cacheKey) + } else { + require.Error(t, err) + require.Contains(t, err.Error(), tc.errMsg) + require.Empty(t, cacheKey) + } + }) + } +} diff --git a/service/cachemdw/middleware_test.go b/service/cachemdw/middleware_test.go new file mode 100644 index 0000000..f3d589e --- /dev/null +++ b/service/cachemdw/middleware_test.go @@ -0,0 +1,110 @@ +package cachemdw_test + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/kava-labs/kava-proxy-service/clients/cache" + "github.com/kava-labs/kava-proxy-service/decode" + "github.com/kava-labs/kava-proxy-service/logging" + "github.com/kava-labs/kava-proxy-service/service" + "github.com/kava-labs/kava-proxy-service/service/cachemdw" +) + +func TestE2ETestServiceCacheMiddleware(t *testing.T) { + logger, err := logging.New("TRACE") + require.NoError(t, err) + + inMemoryCache := cache.NewInMemoryCache() + blockGetter := NewMockEVMBlockGetter() + cacheTTL := time.Duration(0) // TTL: no expiry + + serviceCache := cachemdw.NewServiceCache(inMemoryCache, blockGetter, cacheTTL, service.DecodedRequestContextKey, defaultChainIDString, &logger) + + emptyHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}) + cachingMdw := serviceCache.CachingMiddleware(emptyHandler) + // proxyHandler emulates behaviour of actual service proxy handler + // sequence of execution: + // - isCachedMdw + // - proxyHandler + // - cachingMdw + // - emptyHandler + proxyHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + response := []byte(testEVMQueries[TestRequestEthBlockByNumberSpecific].ResponseBody) + if cachemdw.IsRequestCached(r.Context()) { + w.Header().Add(cachemdw.CacheHeaderKey, cachemdw.CacheHitHeaderValue) + } else { + w.Header().Add(cachemdw.CacheHeaderKey, cachemdw.CacheMissHeaderValue) + } + w.WriteHeader(http.StatusOK) + w.Write(response) + responseContext := context.WithValue(r.Context(), cachemdw.ResponseContextKey, response) + + cachingMdw.ServeHTTP(w, r.WithContext(responseContext)) + }) + isCachedMdw := serviceCache.IsCachedMiddleware(proxyHandler) + + t.Run("cache miss", func(t *testing.T) { + req := createTestHttpRequest( + t, + "https://api.kava.io:8545/thisshouldntshowup", + TestRequestEthBlockByNumberSpecific, + ) + resp := httptest.NewRecorder() + + isCachedMdw.ServeHTTP(resp, req) + + require.Equal(t, http.StatusOK, resp.Code) + require.JSONEq(t, testEVMQueries[TestRequestEthBlockByNumberSpecific].ResponseBody, resp.Body.String()) + require.Equal(t, cachemdw.CacheMissHeaderValue, resp.Header().Get(cachemdw.CacheHeaderKey)) + + cacheItems := inMemoryCache.GetAll(context.Background()) + require.Len(t, cacheItems, 1) + require.Contains(t, cacheItems, "query:1:eth_getBlockByNumber:0x885d3d84b42d647be47d94a001428be7e88ab787251031ddbfb247a581d0505a") + }) + + t.Run("cache hit", func(t *testing.T) { + req := createTestHttpRequest( + t, + "https://api.kava.io:8545/thisshouldntshowup", + TestRequestEthBlockByNumberSpecific, + ) + resp := httptest.NewRecorder() + + isCachedMdw.ServeHTTP(resp, req) + + require.Equal(t, http.StatusOK, resp.Code) + require.JSONEq(t, testEVMQueries[TestRequestEthBlockByNumberSpecific].ResponseBody, resp.Body.String()) + require.Equal(t, cachemdw.CacheHitHeaderValue, resp.Header().Get(cachemdw.CacheHeaderKey)) + }) +} + +func createTestHttpRequest( + t *testing.T, + url string, + reqName testReqName, +) *http.Request { + t.Helper() + + req, err := http.NewRequest(http.MethodGet, url, nil) + require.NoError(t, err) + + decodedReq, err := decode.DecodeEVMRPCRequest( + []byte(testEVMQueries[reqName].RequestBody), + ) + require.NoError(t, err) + + decodedReqCtx := context.WithValue( + req.Context(), + service.DecodedRequestContextKey, + decodedReq, + ) + req = req.WithContext(decodedReqCtx) + + return req +} diff --git a/service/cachemdw/testdata_test.go b/service/cachemdw/testdata_test.go new file mode 100644 index 0000000..cac9f24 --- /dev/null +++ b/service/cachemdw/testdata_test.go @@ -0,0 +1,298 @@ +package cachemdw_test + +type testHttpRequestResponse struct { + RequestBody string + ResponseBody string +} + +type testReqName string + +const ( + TestRequestWeb3ClientVersion testReqName = "web3_clientVersion" + TestRequestEthGetAccountsEmpty testReqName = "eth_getAccounts/empty" + TestRequestEthBlockByNumberSpecific testReqName = "eth_getBlockByNumber" + TestRequestEthBlockByNumberLatest testReqName = "eth_getBlockByNumber/latest" + TestRequestEthBlockByNumberFuture testReqName = "eth_getBlockByNumber/future" + TestRequestEthBlockByNumberError testReqName = "eth_getBlockByNumber/error" + TestRequestEthGetBalancePositive testReqName = "eth_getBalance/positive" + TestRequestEthGetBalanceZero testReqName = "eth_getBalance/zero" + TestRequestEthGetCodeEmpty testReqName = "eth_getCode/empty" +) + +// testEVMQueries is a map of testing json-rpc responses. These are copied from +// real requests to the Kava evm. +var testEVMQueries = map[testReqName]testHttpRequestResponse{ + TestRequestWeb3ClientVersion: { + RequestBody: `{ + "jsonrpc":"2.0", + "method":"web3_clientVersion", + "params":[], + "id":1 + }`, + ResponseBody: `{ + "jsonrpc": "2.0", + "id": 1, + "result": "Version dev ()\nCompiled at using Go go1.20.3 (amd64)" + }`, + }, + TestRequestEthGetAccountsEmpty: { + RequestBody: `{ + "jsonrpc":"2.0", + "method":"eth_accounts", + "params":[], + "id":1 + }`, + ResponseBody: `{ + "jsonrpc": "2.0", + "id": 1, + "result": [] + }`, + }, + TestRequestEthBlockByNumberSpecific: { + RequestBody: `{ + "jsonrpc":"2.0", + "method":"eth_getBlockByNumber", + "params":[ + "0x1b4", + true + ], + "id":1 + }`, + ResponseBody: `{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "difficulty": "0x0", + "extraData": "0x", + "gasLimit": "0x1312d00", + "gasUsed": "0x1afc2", + "hash": "0xcc6963a6d025ec2dad24373fbd5f2c3ab75b51ccb31f049682e2001b1a20322f", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "miner": "0x7f73862f0672c066c3f6b4330a736479f0345cd7", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0x1b4", + "parentHash": "0xd313a81b36d717e4ce67cb7d8f6560158bef9a25f8a4e1b63475050a4181102c", + "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "size": "0x2431", + "stateRoot": "0x20197ba04e30d29a58b508b752d41f0614ceb8d47d2ea2544ff64a6490327625", + "timestamp": "0x628e85a0", + "totalDifficulty": "0x0", + "transactions": [], + "transactionsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "uncles": [] + } + }`, + }, + TestRequestEthBlockByNumberLatest: { + RequestBody: `{ + "jsonrpc":"2.0", + "method":"eth_getBlockByNumber", + "params":[ + "latest", + true + ], + "id":1 + }`, + ResponseBody: `{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "difficulty": "0x0", + "extraData": "0x", + "gasLimit": "0x1312d00", + "gasUsed": "0xffea13", + "hash": "0xe1cbbd4ba91685ce6c3fe51f2a64cc29d81beee2926d803f7f9ba59fba42fb43", + "logsBloom": "0x9030082000000200200000040002300000000000040008000000000000800100800001000040002008000400800080000880021000000802000002001000080000060040800000000140ac09010020200000a0100000000010000200040000004400048042040018008843800a00080080004000000c00001000001108800288000000014080008000001a80000040020400900000020000800201500044210880001000080040c10c081000000000400000040400000480000021001040000000000002000812002084000700430010000000410008028800c20000100020000001000001210040080010000000010240082880400000000000208000004008", + "miner": "0xb21adc77c091742783061ab15a0bd1c27efc7a81", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x0000000000000000", + "number": "0x49be70", + "parentHash": "0x7e30cc8b5f6208d0c07d7964930a8dc5d111e4f5830744121beeb5d028c8332d", + "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "sha3Uncles": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "size": "0x2364", + "stateRoot": "0x992ed1d2a240baf82ac21bd1d143f000181f9d15f0b8f1b03ee33b0b704d32ce", + "timestamp": "0x6465223a", + "totalDifficulty": "0x0", + "transactions": [ + { + "blockHash": "0xe1cbbd4ba91685ce6c3fe51f2a64cc29d81beee2926d803f7f9ba59fba42fb43", + "blockNumber": "0x49be70", + "from": "0xbfa2f9018a41a5419d38bf3e11e8651e998037c5", + "gas": "0x895440", + "gasPrice": "0x1e", + "hash": "0x58dc15c522cce394167619c3d80ed6c7645db4b43b4759d2808e7468be6808cf", + "input": "0xfdb5a03e", + "nonce": "0x5e3c6", + "to": "0x109f3289665a8f034e2cacdbcfb678cabe09f1d5", + "transactionIndex": "0x0", + "value": "0x0", + "type": "0x0", + "chainId": "0x8ae", + "v": "0x1180", + "r": "0x89834b451fd30d4c66e35b14af33d1759541f9758fac889a1fb47dab7759db64", + "s": "0x4511c49cc3ab4dcddcb8e427b3c3b3208c947899e32e461b694efa01d44b2d23" + }, + { + "blockHash": "0xe1cbbd4ba91685ce6c3fe51f2a64cc29d81beee2926d803f7f9ba59fba42fb43", + "blockNumber": "0x49be70", + "from": "0xd479f39e2d2cf61a6708d2a68b245ed04c10683d", + "gas": "0x4c4b40", + "gasPrice": "0x37", + "hash": "0xf71127d678911732c7654d4fcff7b7b690a552e2b1652d5b981a03b93d64bee0", + "input": "0xfdb5a03e", + "nonce": "0x71e2a", + "to": "0xbc50b9f7f8a4ac5cfbb02d214239033dd5a35527", + "transactionIndex": "0x1", + "value": "0x0", + "type": "0x0", + "chainId": "0x8ae", + "v": "0x117f", + "r": "0x19e8dfcca57dbad7359b4cd48347f184a08ab0b939ccf3a6978ec4bedf6507c3", + "s": "0xcd02607f329646b02fff56a211a5442f6c5b13fba6bc01a39edcf6c1d6bdd1e" + }, + { + "blockHash": "0xe1cbbd4ba91685ce6c3fe51f2a64cc29d81beee2926d803f7f9ba59fba42fb43", + "blockNumber": "0x49be70", + "from": "0xbfa2f9018a41a5419d38bf3e11e8651e998037c5", + "gas": "0x895440", + "gasPrice": "0x1e", + "hash": "0x2ee916a9b0732d7badd7d9f5bb7d933bb5344bb672f73bf741b922ff9a9d2252", + "input": "0xfdb5a03e", + "nonce": "0x5e3c7", + "to": "0x738114fc34d7b0d33f13d2b5c3d44484ec85c7f1", + "transactionIndex": "0x2", + "value": "0x0", + "type": "0x0", + "chainId": "0x8ae", + "v": "0x1180", + "r": "0x81dfe4351c448ccc6a5f6f2a2f866ad023df7843da56c38fb19dd0d5d90e22de", + "s": "0x1d770062f304732b4fa5544bd12bd4356f9a853d66bbdef547eab034b142897a" + }, + { + "blockHash": "0xe1cbbd4ba91685ce6c3fe51f2a64cc29d81beee2926d803f7f9ba59fba42fb43", + "blockNumber": "0x49be70", + "from": "0x07f92d445d1fa59059b50fb664a7633b86db1152", + "gas": "0x989680", + "gasPrice": "0x3c", + "hash": "0x9cb2439b6d4784d58118d3facb9beba77dc80fa4c998b281a4cf46e944298c1f", + "input": "0xfdb5a03e", + "nonce": "0x58885", + "to": "0xefa8952a4ab8b210a5f1dd2a378ed3d1200cf64b", + "transactionIndex": "0x3", + "value": "0x0", + "type": "0x0", + "chainId": "0x8ae", + "v": "0x117f", + "r": "0x515203b1e08dc06fa7a1fa4e029775a233aa6f9af419185d0b6b8a6407d27eb5", + "s": "0x52984774ebce17affe5f842ad930605381e5ee146074aa1adc171eb4cc128270" + }, + { + "blockHash": "0xe1cbbd4ba91685ce6c3fe51f2a64cc29d81beee2926d803f7f9ba59fba42fb43", + "blockNumber": "0x49be70", + "from": "0x6d4f641c7f86c5c76182066b7bc1023dfe51c8f0", + "gas": "0x424f3", + "gasPrice": "0x3b9aca00", + "hash": "0x102a60dcd2333651c5f13e53db5ffad994e3fd4d74d99eb1355562da0b1b4d8d", + "input": "0xabe50f1900000000000000000000000000000000000000000000010f0cf064dd592000000000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x376", + "to": "0x2911c3a3b497af71aacbb9b1e9fd3ee5d50f959d", + "transactionIndex": "0x4", + "value": "0x0", + "type": "0x0", + "chainId": "0x8ae", + "v": "0x1180", + "r": "0x1890b8bf21b7a50f4a55568535dcff47d89a257e780e51e419c237af943f2afe", + "s": "0x313b04a2e0ea400623fbcfd22af91ebd2593c3fdbd29fbe1004e81e54430770a" + } + ], + "transactionsRoot": "0xfa3bae7d2ee5eff10fe2ff44840d31755c56eb0dfd0827ebc5ad21ac628020d3", + "uncles": [] + } + }`, + }, + TestRequestEthBlockByNumberFuture: { + RequestBody: `{ + "jsonrpc":"2.0", + "method":"eth_getBlockByNumber", + "params":[ + "0x59be70", + true + ], + "id":1 + }`, + ResponseBody: `{ + "jsonrpc": "2.0", + "id": 1, + "result": null + }`, + }, + TestRequestEthBlockByNumberError: { + RequestBody: `{ + "jsonrpc":"2.0", + "method":"eth_getBlockByNumber", + "params":[ + oops + ], + "id":1 + }`, + ResponseBody: `{ + "jsonrpc": "2.0", + "id": null, + "error": { + "code": -32700, + "message": "parse error" + } + }`, + }, + TestRequestEthGetBalancePositive: { + RequestBody: `{ + "jsonrpc":"2.0", + "method":"eth_getBalance", + "params":[ + "0x373CE80dd1e921506EC5603290AF444e60CeF61F", + "0x49BCF0" + ], + "id":1 + }`, + ResponseBody: `{ + "jsonrpc": "2.0", + "id": 1, + "result": "0xdfe3285d58c7e365" + }`, + }, + TestRequestEthGetBalanceZero: { + RequestBody: `{ + "jsonrpc":"2.0", + "method":"eth_getBalance", + "params":[ + "0x1111111111111111111111111111111111111111", + "0x2" + ], + "id":1 + }`, + ResponseBody: `{ + "jsonrpc": "2.0", + "id": 1, + "result": "0x0" + }`, + }, + TestRequestEthGetCodeEmpty: { + RequestBody: `{ + "jsonrpc":"2.0", + "method":"eth_getCode", + "params":[ + "0x1111111111111111111111111111111111111111", + "0x2" + ], + "id":1 + }`, + ResponseBody: `{ + "jsonrpc": "2.0", + "id": 1, + "result": "0x" + }`, + }, +} diff --git a/service/handlers.go b/service/handlers.go index 338071e..da2fa3c 100644 --- a/service/handlers.go +++ b/service/handlers.go @@ -1,6 +1,7 @@ package service import ( + "context" "encoding/json" "errors" "fmt" @@ -20,12 +21,22 @@ func createHealthcheckHandler(service *ProxyService) func(http.ResponseWriter, * // check that the database is reachable err := service.Database.HealthCheck() - if err != nil { errMsg := fmt.Errorf("proxy service unable to connect to database") combinedErrors = errors.Join(combinedErrors, errMsg) } + // check that the cache is reachable + err = service.Cache.Healthcheck(context.Background()) + if err != nil { + service.Logger.Error(). + Err(err). + Msg("cache healthcheck failed") + + errMsg := fmt.Errorf("proxy service unable to connect to cache") + combinedErrors = errors.Join(combinedErrors, errMsg) + } + if combinedErrors != nil { w.WriteHeader(http.StatusInternalServerError) diff --git a/service/middleware.go b/service/middleware.go index b77f2f1..82ae226 100644 --- a/service/middleware.go +++ b/service/middleware.go @@ -16,6 +16,7 @@ import ( "github.com/kava-labs/kava-proxy-service/config" "github.com/kava-labs/kava-proxy-service/decode" "github.com/kava-labs/kava-proxy-service/logging" + "github.com/kava-labs/kava-proxy-service/service/cachemdw" ) const ( @@ -228,8 +229,35 @@ func createProxyRequestMiddleware(next http.Handler, config config.Config, servi serviceLogger.Trace().Msg("request body is empty, skipping before request interceptors") } - // proxy request to backend origin servers - proxy.ServeHTTP(lrw, r) + isCached := cachemdw.IsRequestCached(r.Context()) + response := r.Context().Value(cachemdw.ResponseContextKey) + typedResponse, ok := response.([]byte) + + // if request is cached and response is present in context - serve the request from the cache + // otherwise proxy to the actual backend + if isCached && ok { + serviceLogger.Logger.Trace(). + Str("method", r.Method). + Str("url", r.URL.String()). + Str("host", r.Host). + Msg("cache hit") + + w.Header().Add(cachemdw.CacheHeaderKey, cachemdw.CacheHitHeaderValue) + w.Header().Add("Content-Type", "application/json") + _, err := w.Write(typedResponse) + if err != nil { + serviceLogger.Logger.Error().Msg(fmt.Sprintf("can't write cached response: %v", err)) + } + } else { + serviceLogger.Logger.Trace(). + Str("method", r.Method). + Str("url", r.URL.String()). + Str("host", r.Host). + Msg("cache miss") + + w.Header().Add(cachemdw.CacheHeaderKey, cachemdw.CacheMissHeaderValue) + proxy.ServeHTTP(lrw, r) + } serviceLogger.Trace().Msg(fmt.Sprintf("response %+v \nheaders %+v \nstatus %+v for request %+v", lrw.Status(), lrw.Header(), lrw.body, r)) @@ -243,7 +271,20 @@ func createProxyRequestMiddleware(next http.Handler, config config.Config, servi // extract the original hostname the request was sent to requestHostnameContext := context.WithValue(originRoundtripLatencyContext, RequestHostnameContextKey, r.Host) - enrichedContext := requestHostnameContext + var bodyCopy bytes.Buffer + tee := io.TeeReader(lrw.body, &bodyCopy) + // read all body from reader into bodyBytes, and copy into bodyCopy + bodyBytes, err := io.ReadAll(tee) + if err != nil { + serviceLogger.Error().Err(err).Msg("can't read lrw.body") + } + + // replace empty body reader with fresh copy + lrw.body = &bodyCopy + // set body in context + responseContext := context.WithValue(requestHostnameContext, cachemdw.ResponseContextKey, bodyBytes) + + enrichedContext := responseContext // parse the remote address of the request for use below remoteAddressParts := strings.Split(r.RemoteAddr, ":") @@ -383,6 +424,7 @@ func createAfterProxyFinalizer(service *ProxyService, config config.Config) http } var blockNumber *int64 + // TODO: Redundant ExtractBlockNumberFromEVMRPCRequest call here if request is cached rawBlockNumber, err := decodedRequestBody.ExtractBlockNumberFromEVMRPCRequest(r.Context(), service.evmClient) if err != nil { diff --git a/service/service.go b/service/service.go index ac91e82..26637a9 100644 --- a/service/service.go +++ b/service/service.go @@ -9,15 +9,18 @@ import ( "time" "github.com/ethereum/go-ethereum/ethclient" + "github.com/kava-labs/kava-proxy-service/clients/cache" "github.com/kava-labs/kava-proxy-service/clients/database" "github.com/kava-labs/kava-proxy-service/clients/database/migrations" "github.com/kava-labs/kava-proxy-service/config" "github.com/kava-labs/kava-proxy-service/logging" + "github.com/kava-labs/kava-proxy-service/service/cachemdw" ) // ProxyService represents an instance of the proxy service API type ProxyService struct { Database *database.PostgresClient + Cache *cachemdw.ServiceCache httpProxy *http.Server evmClient *ethclient.Client *logging.ServiceLogger @@ -27,6 +30,24 @@ type ProxyService struct { func New(ctx context.Context, config config.Config, serviceLogger *logging.ServiceLogger) (ProxyService, error) { service := ProxyService{} + // create database client + db, err := createDatabaseClient(ctx, config, serviceLogger) + if err != nil { + return ProxyService{}, err + } + + // create evm api client + evmClient, err := ethclient.Dial(config.EvmQueryServiceURL) + if err != nil { + return ProxyService{}, err + } + + // create cache client + serviceCache, err := createServiceCache(ctx, config, serviceLogger, evmClient) + if err != nil { + return ProxyService{}, err + } + // create an http router for registering handlers for a given route mux := http.NewServeMux() @@ -38,12 +59,24 @@ func New(ctx context.Context, config config.Config, serviceLogger *logging.Servi // set up before and after request interceptors (a.k.a. raptors 🦖🦖) + // CachingMiddleware caches request in case of: + // - request isn't already cached + // - request is cacheable + // - response is present in context + cacheAfterProxyMiddleware := serviceCache.CachingMiddleware(afterProxyFinalizer) + // create an http handler that will proxy any request to the specified URL - proxyMiddleware := createProxyRequestMiddleware(afterProxyFinalizer, config, serviceLogger, []RequestInterceptor{}, []RequestInterceptor{}) + proxyMiddleware := createProxyRequestMiddleware(cacheAfterProxyMiddleware, config, serviceLogger, []RequestInterceptor{}, []RequestInterceptor{}) + + // IsCachedMiddleware works in the following way: + // - tries to get response from the cache + // - if present sets cached response in context, marks as cached in context and forwards to next middleware + // - if not present marks as uncached in context and forwards to next middleware + cacheMiddleware := serviceCache.IsCachedMiddleware(proxyMiddleware) // create an http handler that will log the request to stdout // this handler will run before the proxyMiddleware handler - requestLoggingMiddleware := createRequestLoggingMiddleware(proxyMiddleware, serviceLogger) + requestLoggingMiddleware := createRequestLoggingMiddleware(cacheMiddleware, serviceLogger) // register healthcheck handler that can be used during deployment and operations // to determine if the service is ready to receive requests @@ -64,13 +97,6 @@ func New(ctx context.Context, config config.Config, serviceLogger *logging.Servi ReadTimeout: time.Duration(config.HTTPReadTimeoutSeconds) * time.Second, } - // create database client - db, err := createDatabaseClient(ctx, config, serviceLogger) - - if err != nil { - return ProxyService{}, err - } - // register database status handler // for responding to requests for the status // of database related operations such as @@ -78,17 +104,11 @@ func New(ctx context.Context, config config.Config, serviceLogger *logging.Servi // partitioning mux.HandleFunc("/status/database", createDatabaseStatusHandler(&service, db)) - // create evm api client - evmClient, err := ethclient.Dial(config.EvmQueryServiceURL) - - if err != nil { - return ProxyService{}, err - } - service = ProxyService{ httpProxy: server, ServiceLogger: serviceLogger, Database: db, + Cache: serviceCache, evmClient: evmClient, } @@ -167,6 +187,38 @@ func createDatabaseClient(ctx context.Context, config config.Config, logger *log return &serviceDatabase, err } +func createServiceCache( + ctx context.Context, + config config.Config, + logger *logging.ServiceLogger, + evmclient *ethclient.Client, +) (*cachemdw.ServiceCache, error) { + cfg := cache.RedisConfig{ + Address: config.RedisEndpointURL, + Password: config.RedisPassword, + DB: 0, + } + redisCache, err := cache.NewRedisCache( + &cfg, + logger, + ) + if err != nil { + logger.Error().Msg(fmt.Sprintf("error %s creating cache using endpoint %+v", err, config.RedisEndpointURL)) + return nil, err + } + + serviceCache := cachemdw.NewServiceCache( + redisCache, + evmclient, + config.CacheTTL, + DecodedRequestContextKey, + config.ChainID, + logger, + ) + + return serviceCache, nil +} + // Run runs the proxy service, returning error (if any) in the event // the proxy service stops func (p *ProxyService) Run() error { From 15a5e50134dd961dec6afeb45f546230251275ed Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Tue, 17 Oct 2023 11:08:53 -0400 Subject: [PATCH 02/29] Minor fix --- main_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/main_test.go b/main_test.go index b729d24..819337f 100644 --- a/main_test.go +++ b/main_test.go @@ -593,10 +593,6 @@ func cleanUpRedis(t *testing.T, redisClient *redis.Client) { keys, err := redisClient.Keys(context.Background(), "*").Result() require.NoError(t, err) - for _, key := range keys { - fmt.Printf("key: %v\n", key) - } - if len(keys) != 0 { _, err = redisClient.Del(context.Background(), keys...).Result() require.NoError(t, err) From 9a431d98406f0edcb4e1ebddad0cb8e81deb536a Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Tue, 17 Oct 2023 12:57:15 -0400 Subject: [PATCH 03/29] Added comments --- service/cachemdw/cache.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/service/cachemdw/cache.go b/service/cachemdw/cache.go index ef25f25..ce26e82 100644 --- a/service/cachemdw/cache.go +++ b/service/cachemdw/cache.go @@ -57,10 +57,13 @@ func IsCacheable( return false } + // blockNumber <= 0 means magic tag was used, one of the "latest", "pending", "earliest", etc... + // as of now we don't cache requests with magic tags if blockNumber <= 0 { return false } + // block number is specified and it's not a magic tag - cache the request return true } From 1f5e85e985bc002c6f0a7994af8b3063445fa006 Mon Sep 17 00:00:00 2001 From: Evgeniy Scherbina Date: Tue, 17 Oct 2023 13:46:59 -0400 Subject: [PATCH 04/29] Update CACHING.md --- architecture/CACHING.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/architecture/CACHING.md b/architecture/CACHING.md index 218e762..5a2f516 100644 --- a/architecture/CACHING.md +++ b/architecture/CACHING.md @@ -32,6 +32,23 @@ package provides two different middlewares: - if not present marks as uncached in context and forwards to next middleware - next middleware should check whether request was cached and act accordingly: +## What requests are cached? + +As of now we only cache requests which has `specified block number` in params +Requests without block number or with a magic word instead of + +`{ + "jsonrpc":"2.0", + "method":"eth_getBalance", + "params":[ + "0x373CE80dd1e921506EC5603290AF444e60CeF61F", + "0x49BCF0" + ], + "id":1 + }`, + + TODO... + ## Cache Invalidation ### Keys Structure From de33eb9729630d81a20e3ebf18df05101ce33a8e Mon Sep 17 00:00:00 2001 From: Evgeniy Scherbina Date: Tue, 17 Oct 2023 14:36:56 -0400 Subject: [PATCH 05/29] Update CACHING.md --- architecture/CACHING.md | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/architecture/CACHING.md b/architecture/CACHING.md index 5a2f516..389516c 100644 --- a/architecture/CACHING.md +++ b/architecture/CACHING.md @@ -34,20 +34,17 @@ package provides two different middlewares: ## What requests are cached? -As of now we only cache requests which has `specified block number` in params -Requests without block number or with a magic word instead of - -`{ - "jsonrpc":"2.0", - "method":"eth_getBalance", - "params":[ - "0x373CE80dd1e921506EC5603290AF444e60CeF61F", - "0x49BCF0" - ], - "id":1 - }`, - - TODO... +As of now we cache requests which has `specific block number` in request, for example: +```json +{ + "jsonrpc":"2.0", + "method":"eth_getBlockByNumber", + "params":["0x1b4", true], + "id":1 +} +``` + +we don't cache requests without `specific block number` or requests which uses magic tags as a block number: "latest", "pending", "earliest", etc... ## Cache Invalidation From f5cb0c50d7074a0f79d6790d9f8af4e5bce00cdc Mon Sep 17 00:00:00 2001 From: Evgeniy Scherbina Date: Tue, 17 Oct 2023 14:37:41 -0400 Subject: [PATCH 06/29] Update CACHING.md --- architecture/CACHING.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/architecture/CACHING.md b/architecture/CACHING.md index 389516c..355ebb2 100644 --- a/architecture/CACHING.md +++ b/architecture/CACHING.md @@ -39,7 +39,10 @@ As of now we cache requests which has `specific block number` in request, for ex { "jsonrpc":"2.0", "method":"eth_getBlockByNumber", - "params":["0x1b4", true], + "params":[ + "0x1b4", // specific block number + true + ], "id":1 } ``` From b1407ecdcf188b125a54148e0af2d4c0d935b53d Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 18 Oct 2023 12:13:18 -0400 Subject: [PATCH 07/29] Add response parser --- service/cachemdw/response.go | 99 +++++++++++++++++++++++++++ service/cachemdw/response_test.go | 109 ++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 service/cachemdw/response.go create mode 100644 service/cachemdw/response_test.go diff --git a/service/cachemdw/response.go b/service/cachemdw/response.go new file mode 100644 index 0000000..d4f9dcd --- /dev/null +++ b/service/cachemdw/response.go @@ -0,0 +1,99 @@ +package cachemdw + +import ( + "encoding/json" + "errors" + "fmt" +) + +type JsonRpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// String returns the string representation of the error +func (e *JsonRpcError) String() string { + return fmt.Sprintf("%s (code: %d)", e.Message, e.Code) +} + +// JsonRpcResponse is a EVM JSON-RPC response +type JsonRpcResponse struct { + Version string `json:"jsonrpc,omitempty"` + ID json.RawMessage `json:"id,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + JsonRpcError *JsonRpcError `json:"error,omitempty"` +} + +// UnmarshalJsonRpcResponse unmarshals a JSON-RPC response +func UnmarshalJsonRpcResponse(data []byte) (*JsonRpcResponse, error) { + var msg JsonRpcResponse + err := json.Unmarshal(data, &msg) + return &msg, err +} + +// Marshal marshals a JSON-RPC response to JSON +func (resp *JsonRpcResponse) Marshal() ([]byte, error) { + return json.Marshal(resp) +} + +// Error returns the json-rpc error if any +func (resp *JsonRpcResponse) Error() error { + if resp.JsonRpcError == nil { + return nil + } + + return errors.New(resp.JsonRpcError.String()) +} + +// IsResultEmpty checks if the response's result is empty +func (resp *JsonRpcResponse) IsResultEmpty() bool { + if len(resp.Result) == 0 { + // empty response's result + return true + } + + var result interface{} + err := json.Unmarshal(resp.Result, &result) + if err != nil { + // consider result as empty if it's malformed + return true + } + + switch r := result.(type) { + case []interface{}: + // consider result as empty if it's empty slice + return len(r) == 0 + case string: + // Matches: + // - "" - Empty string + // - "0x0" - Represents zero in official json-rpc conventions. See: + // https://ethereum.org/en/developers/docs/apis/json-rpc/#conventions + // + // - "0x" - Empty response from some endpoints like getCode + + return r == "" || r == "0x0" || r == "0x" + case bool: + // consider result as empty if it's false + return !r + case nil: + // consider result as empty if it's null + return true + default: + return false + } +} + +// IsCacheable returns true in case of: +// - json-rpc response doesn't contain an error +// - json-rpc response's result isn't empty +func (resp *JsonRpcResponse) IsCacheable() bool { + if err := resp.Error(); err != nil { + return false + } + + if resp.IsResultEmpty() { + return false + } + + return true +} diff --git a/service/cachemdw/response_test.go b/service/cachemdw/response_test.go new file mode 100644 index 0000000..63d1b43 --- /dev/null +++ b/service/cachemdw/response_test.go @@ -0,0 +1,109 @@ +package cachemdw_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kava-labs/kava-proxy-service/service/cachemdw" +) + +func TestUnitTestJsonRpcResponse_IsEmpty(t *testing.T) { + toJSON := func(t *testing.T, result any) []byte { + resultInJSON, err := json.Marshal(result) + require.NoError(t, err) + + return resultInJSON + } + + mkResp := func(result []byte) *cachemdw.JsonRpcResponse { + return &cachemdw.JsonRpcResponse{ + Version: "2.0", + ID: []byte("1"), + Result: result, + } + } + + tests := []struct { + name string + resp *cachemdw.JsonRpcResponse + isEmpty bool + }{ + { + name: "empty result", + resp: mkResp([]byte("")), + isEmpty: true, + }, + { + name: "invalid json", + resp: mkResp([]byte("invalid json")), + isEmpty: true, + }, + { + name: "empty slice", + resp: mkResp(toJSON(t, []interface{}{})), + isEmpty: true, + }, + { + name: "empty string", + resp: mkResp(toJSON(t, "")), + isEmpty: true, + }, + { + name: "0x0 string", + resp: mkResp(toJSON(t, "0x0")), + isEmpty: true, + }, + { + name: "0x string", + resp: mkResp(toJSON(t, "0x")), + isEmpty: true, + }, + { + name: "empty bool", + resp: mkResp(toJSON(t, false)), + isEmpty: true, + }, + { + name: "nil", + resp: mkResp(nil), + isEmpty: true, + }, + { + name: "null", + resp: mkResp(toJSON(t, nil)), + isEmpty: true, + }, + { + name: "non-empty slice", + resp: mkResp(toJSON(t, []interface{}{1})), + isEmpty: false, + }, + { + name: "non-empty string", + resp: mkResp(toJSON(t, "0x1234")), + isEmpty: false, + }, + { + name: "non-empty bool", + resp: mkResp(toJSON(t, true)), + isEmpty: false, + }, + { + name: "unsupported empty object", + resp: mkResp(toJSON(t, map[string]interface{}{})), + isEmpty: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal( + t, + tc.isEmpty, + tc.resp.IsResultEmpty(), + ) + }) + } +} From ec6a41c554fe2eb55a4d074016754619a7fca4bb Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 18 Oct 2023 14:40:35 -0400 Subject: [PATCH 08/29] Integrate response parser and adjust tests accordingly --- main_test.go | 108 ++++++++++++++++++++++++++++---------- service/cachemdw/cache.go | 14 +++-- 2 files changed, 91 insertions(+), 31 deletions(-) diff --git a/main_test.go b/main_test.go index 819337f..40a96c3 100644 --- a/main_test.go +++ b/main_test.go @@ -483,10 +483,7 @@ func TestE2ETestProxyTracksBlockNumberForMethodsWithBlockHashParam(t *testing.T) } } -func TestE2ETestProxyCachesMethodsWithBlockNumberParam(t *testing.T) { - testRandomAddressHex := "0x6767114FFAA17C6439D7AEA480738B982CE63A02" - testAddress := common.HexToAddress(testRandomAddressHex) - +func TestE2eTestCachingMdwWithBlockNumberParam(t *testing.T) { // create api and database clients client, err := ethclient.Dial(proxyServiceURL) if err != nil { @@ -509,15 +506,9 @@ func TestE2ETestProxyCachesMethodsWithBlockNumberParam(t *testing.T) { }{ { desc: "test case #1", - method: "eth_getTransactionCount", - params: []interface{}{testAddress, "0x1"}, - keysNum: 1, - }, - { - desc: "test case #2", method: "eth_getBlockByNumber", params: []interface{}{"0x1", true}, - keysNum: 2, + keysNum: 1, }, } { t.Run(tc.desc, func(t *testing.T) { @@ -547,36 +538,97 @@ func TestE2ETestProxyCachesMethodsWithBlockNumberParam(t *testing.T) { }) } - // test cache MISS and cache HIT scenarios for eth_getTransactionCount method + // test cache MISS and cache HIT scenarios for eth_getBlockByNumber method // check that cached and non-cached responses are equal { - // eth_getTransactionCount - cache MISS - bal1, err := client.NonceAt(testContext, testAddress, big.NewInt(2)) + // eth_getBlockByNumber - cache MISS + block1, err := client.BlockByNumber(testContext, big.NewInt(2)) require.NoError(t, err) - expectKeysNum(t, redisClient, 3) + expectKeysNum(t, redisClient, 2) - // eth_getTransactionCount - cache HIT - bal2, err := client.NonceAt(testContext, testAddress, big.NewInt(2)) + // eth_getBlockByNumber - cache HIT + block2, err := client.BlockByNumber(testContext, big.NewInt(2)) require.NoError(t, err) - expectKeysNum(t, redisClient, 3) + expectKeysNum(t, redisClient, 2) - require.Equal(t, bal1, bal2, "balances should be the same") + require.Equal(t, block1, block2, "blocks should be the same") } - // test cache MISS and cache HIT scenarios for eth_getBlockByNumber method - // check that cached and non-cached responses are equal + cleanUpRedis(t, redisClient) +} + +func TestE2eTestCachingMdwWithBlockNumberParam_EmptyResult(t *testing.T) { + testRandomAddressHex := "0x6767114FFAA17C6439D7AEA480738B982CE63A02" + testAddress := common.HexToAddress(testRandomAddressHex) + + // create api and database clients + client, err := ethclient.Dial(proxyServiceURL) + if err != nil { + t.Fatal(err) + } + + redisClient := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("localhost:%v", redisHostPort), + Password: redisPassword, + DB: 0, + }) + cleanUpRedis(t, redisClient) + expectKeysNum(t, redisClient, 0) + + for _, tc := range []struct { + desc string + method string + params []interface{} + keysNum int + }{ + { + desc: "test case #1", + method: "eth_getTransactionCount", + params: []interface{}{testAddress, "0x1"}, + keysNum: 0, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + // both calls should lead to cache MISS scenario, because empty results aren't cached + // check corresponding values in cachemdw.CacheMissHeaderValue HTTP header + // check that responses are equal + + // eth_getBlockByNumber - cache MISS + resp1 := mkJsonRpcRequest(t, proxyServiceURL, tc.method, tc.params) + require.Equal(t, cachemdw.CacheMissHeaderValue, resp1.Header[cachemdw.CacheHeaderKey][0]) + body1, err := io.ReadAll(resp1.Body) + require.NoError(t, err) + err = checkJsonRpcErr(body1) + require.NoError(t, err) + expectKeysNum(t, redisClient, tc.keysNum) + + // eth_getBlockByNumber - cache MISS again (empty results aren't cached) + resp2 := mkJsonRpcRequest(t, proxyServiceURL, tc.method, tc.params) + require.Equal(t, cachemdw.CacheMissHeaderValue, resp2.Header[cachemdw.CacheHeaderKey][0]) + body2, err := io.ReadAll(resp2.Body) + require.NoError(t, err) + err = checkJsonRpcErr(body2) + require.NoError(t, err) + expectKeysNum(t, redisClient, tc.keysNum) + + require.JSONEq(t, string(body1), string(body2), "blocks should be the same") + }) + } + + // both calls should lead to cache MISS scenario, because empty results aren't cached + // check that responses are equal { - // eth_getBlockByNumber - cache MISS - block1, err := client.BlockByNumber(testContext, big.NewInt(2)) + // eth_getTransactionCount - cache MISS + bal1, err := client.NonceAt(testContext, testAddress, big.NewInt(2)) require.NoError(t, err) - expectKeysNum(t, redisClient, 4) + expectKeysNum(t, redisClient, 0) - // eth_getBlockByNumber - cache HIT - block2, err := client.BlockByNumber(testContext, big.NewInt(2)) + // eth_getTransactionCount - cache MISS again (empty results aren't cached) + bal2, err := client.NonceAt(testContext, testAddress, big.NewInt(2)) require.NoError(t, err) - expectKeysNum(t, redisClient, 4) + expectKeysNum(t, redisClient, 0) - require.Equal(t, block1, block2, "blocks should be the same") + require.Equal(t, bal1, bal2, "balances should be the same") } cleanUpRedis(t, redisClient) diff --git a/service/cachemdw/cache.go b/service/cachemdw/cache.go index ce26e82..f2326bf 100644 --- a/service/cachemdw/cache.go +++ b/service/cachemdw/cache.go @@ -3,6 +3,7 @@ package cachemdw import ( "context" "errors" + "fmt" "time" "github.com/kava-labs/kava-proxy-service/clients/cache" @@ -107,15 +108,22 @@ func (c *ServiceCache) CacheQueryResponse( func (c *ServiceCache) ValidateAndCacheQueryResponse( ctx context.Context, req *decode.EVMRPCRequestEnvelope, - response []byte, + responseInBytes []byte, ) error { - // TODO(yevhenii): add validation + response, err := UnmarshalJsonRpcResponse(responseInBytes) + if err != nil { + return fmt.Errorf("can't unmarshal json-rpc response: %w", err) + } + // don't cache uncacheable responses + if !response.IsCacheable() { + return fmt.Errorf("response isn't cacheable") + } if err := c.CacheQueryResponse( ctx, req, c.chainID, - response, + responseInBytes, ); err != nil { return err } From 18c59bf49ff6909821603f006c6d5d4e70dfca99 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 18 Oct 2023 14:53:29 -0400 Subject: [PATCH 09/29] Fix tests --- service/cachemdw/cache_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/cachemdw/cache_test.go b/service/cachemdw/cache_test.go index 4675145..7025995 100644 --- a/service/cachemdw/cache_test.go +++ b/service/cachemdw/cache_test.go @@ -25,7 +25,7 @@ const ( var ( defaultChainID = big.NewInt(1) - defaultQueryResp = []byte("resp") + defaultQueryResp = []byte(testEVMQueries[TestRequestWeb3ClientVersion].ResponseBody) ) type MockEVMBlockGetter struct{} From 199fb9a2789b3f3d2f0c2a57f9f4e5efd6412608 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 18 Oct 2023 15:05:45 -0400 Subject: [PATCH 10/29] Improve tests --- service/cachemdw/response_test.go | 59 ++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/service/cachemdw/response_test.go b/service/cachemdw/response_test.go index 63d1b43..978be5a 100644 --- a/service/cachemdw/response_test.go +++ b/service/cachemdw/response_test.go @@ -9,7 +9,7 @@ import ( "github.com/kava-labs/kava-proxy-service/service/cachemdw" ) -func TestUnitTestJsonRpcResponse_IsEmpty(t *testing.T) { +func TestUnitTestJsonRpcResponse_IsResultEmpty(t *testing.T) { toJSON := func(t *testing.T, result any) []byte { resultInJSON, err := json.Marshal(result) require.NoError(t, err) @@ -107,3 +107,60 @@ func TestUnitTestJsonRpcResponse_IsEmpty(t *testing.T) { }) } } + +func TestUnitTestJsonRpcResponse_IsCacheable(t *testing.T) { + toJSON := func(t *testing.T, result any) []byte { + resultInJSON, err := json.Marshal(result) + require.NoError(t, err) + + return resultInJSON + } + + tests := []struct { + name string + resp *cachemdw.JsonRpcResponse + isCacheable bool + }{ + { + name: "empty result", + resp: &cachemdw.JsonRpcResponse{ + Version: "2.0", + ID: []byte("1"), + Result: []byte{}, + }, + isCacheable: false, + }, + { + name: "non-empty error", + resp: &cachemdw.JsonRpcResponse{ + Version: "2.0", + ID: []byte("1"), + Result: toJSON(t, "0x1234"), + JsonRpcError: &cachemdw.JsonRpcError{ + Code: 1, + Message: "error", + }, + }, + isCacheable: false, + }, + { + name: "valid response", + resp: &cachemdw.JsonRpcResponse{ + Version: "2.0", + ID: []byte("1"), + Result: toJSON(t, "0x1234"), + }, + isCacheable: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal( + t, + tc.isCacheable, + tc.resp.IsCacheable(), + ) + }) + } +} From 8039d9560f0bdb482103357a6b09cd7d912d4f35 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 18 Oct 2023 15:47:10 -0400 Subject: [PATCH 11/29] CR's fixes --- service/cachemdw/cache.go | 27 +++++--------------------- service/cachemdw/cache_test.go | 26 +------------------------ service/cachemdw/caching_middleware.go | 2 +- 3 files changed, 7 insertions(+), 48 deletions(-) diff --git a/service/cachemdw/cache.go b/service/cachemdw/cache.go index f2326bf..b41ade8 100644 --- a/service/cachemdw/cache.go +++ b/service/cachemdw/cache.go @@ -90,26 +90,13 @@ func (c *ServiceCache) GetCachedQueryResponse( func (c *ServiceCache) CacheQueryResponse( ctx context.Context, req *decode.EVMRPCRequestEnvelope, - chainID string, - response []byte, + responseInBytes []byte, ) error { + // don't cache uncacheable requests if !IsCacheable(ctx, c.blockGetter, c.ServiceLogger, req) { return errors.New("query isn't cacheable") } - key, err := GetQueryKey(chainID, req) - if err != nil { - return err - } - - return c.cacheClient.Set(ctx, key, response, c.cacheTTL) -} - -func (c *ServiceCache) ValidateAndCacheQueryResponse( - ctx context.Context, - req *decode.EVMRPCRequestEnvelope, - responseInBytes []byte, -) error { response, err := UnmarshalJsonRpcResponse(responseInBytes) if err != nil { return fmt.Errorf("can't unmarshal json-rpc response: %w", err) @@ -119,16 +106,12 @@ func (c *ServiceCache) ValidateAndCacheQueryResponse( return fmt.Errorf("response isn't cacheable") } - if err := c.CacheQueryResponse( - ctx, - req, - c.chainID, - responseInBytes, - ); err != nil { + key, err := GetQueryKey(c.chainID, req) + if err != nil { return err } - return nil + return c.cacheClient.Set(ctx, key, responseInBytes, c.cacheTTL) } func (c *ServiceCache) Healthcheck(ctx context.Context) error { diff --git a/service/cachemdw/cache_test.go b/service/cachemdw/cache_test.go index 7025995..626422a 100644 --- a/service/cachemdw/cache_test.go +++ b/service/cachemdw/cache_test.go @@ -90,31 +90,7 @@ func TestUnitTestCacheQueryResponse(t *testing.T) { require.Equal(t, cache.ErrNotFound, err) require.Empty(t, resp) - err = serviceCache.CacheQueryResponse(ctxb, req, defaultChainIDString, defaultQueryResp) - require.NoError(t, err) - - resp, err = serviceCache.GetCachedQueryResponse(ctxb, req) - require.NoError(t, err) - require.Equal(t, defaultQueryResp, resp) -} - -func TestUnitTestValidateAndCacheQueryResponse(t *testing.T) { - logger, err := logging.New("TRACE") - require.NoError(t, err) - - inMemoryCache := cache.NewInMemoryCache() - blockGetter := NewMockEVMBlockGetter() - cacheTTL := time.Hour - ctxb := context.Background() - - serviceCache := cachemdw.NewServiceCache(inMemoryCache, blockGetter, cacheTTL, service.DecodedRequestContextKey, defaultChainIDString, &logger) - - req := mkEVMRPCRequestEnvelope(defaultBlockNumber) - resp, err := serviceCache.GetCachedQueryResponse(ctxb, req) - require.Equal(t, cache.ErrNotFound, err) - require.Empty(t, resp) - - err = serviceCache.ValidateAndCacheQueryResponse(ctxb, req, defaultQueryResp) + err = serviceCache.CacheQueryResponse(ctxb, req, defaultQueryResp) require.NoError(t, err) resp, err = serviceCache.GetCachedQueryResponse(ctxb, req) diff --git a/service/cachemdw/caching_middleware.go b/service/cachemdw/caching_middleware.go index 25f1ae2..ec41d9f 100644 --- a/service/cachemdw/caching_middleware.go +++ b/service/cachemdw/caching_middleware.go @@ -40,7 +40,7 @@ func (c *ServiceCache) CachingMiddleware( // if request isn't already cached, request is cacheable and response is present in context - cache the response if !isCached && cacheable && ok { - if err := c.ValidateAndCacheQueryResponse( + if err := c.CacheQueryResponse( r.Context(), decodedReq, typedResponse, From 40d0c598a646111a5b3eb316edc1fc15269821f7 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 18 Oct 2023 17:04:02 -0400 Subject: [PATCH 12/29] CR's fixes --- config/config.go | 10 +++++++--- service/cachemdw/cache.go | 12 ++++++------ service/cachemdw/cache_test.go | 15 +++++---------- service/cachemdw/keys.go | 4 ++-- service/cachemdw/keys_test.go | 16 ++++++++-------- service/cachemdw/middleware_test.go | 2 +- service/service.go | 2 +- 7 files changed, 30 insertions(+), 31 deletions(-) diff --git a/config/config.go b/config/config.go index 4891135..2963d37 100644 --- a/config/config.go +++ b/config/config.go @@ -39,8 +39,12 @@ type Config struct { MetricPartitioningPrefillPeriodDays int RedisEndpointURL string RedisPassword string - CacheTTL time.Duration - ChainID string + // TTL for cached evm requests + CacheTTL time.Duration + // CachePrefix is used as prefix for any key in the cache, key has such structure: + // query::: + // Possible values are testnet, mainnet, etc... + CachePrefix string } const ( @@ -216,6 +220,6 @@ func ReadConfig() Config { RedisEndpointURL: os.Getenv(REDIS_ENDPOINT_URL_ENVIRONMENT_KEY), RedisPassword: os.Getenv(REDIS_PASSWORD_ENVIRONMENT_KEY), CacheTTL: time.Duration(EnvOrDefaultInt(CACHE_TTL_ENVIRONMENT_KEY, DEFAULT_CACHE_TTL_SECONDS)) * time.Second, - ChainID: os.Getenv(CHAIN_ID_ENVIRONMENT_KEY), + CachePrefix: os.Getenv(CHAIN_ID_ENVIRONMENT_KEY), } } diff --git a/service/cachemdw/cache.go b/service/cachemdw/cache.go index b41ade8..7ecaa11 100644 --- a/service/cachemdw/cache.go +++ b/service/cachemdw/cache.go @@ -18,8 +18,8 @@ type ServiceCache struct { blockGetter decode.EVMBlockGetter cacheTTL time.Duration decodedRequestContextKey any - // chainID is used as prefix for any key in the cache - chainID string + // cachePrefix is used as prefix for any key in the cache + cachePrefix string *logging.ServiceLogger } @@ -29,7 +29,7 @@ func NewServiceCache( blockGetter decode.EVMBlockGetter, cacheTTL time.Duration, decodedRequestContextKey any, - chainID string, + cachePrefix string, logger *logging.ServiceLogger, ) *ServiceCache { return &ServiceCache{ @@ -37,7 +37,7 @@ func NewServiceCache( blockGetter: blockGetter, cacheTTL: cacheTTL, decodedRequestContextKey: decodedRequestContextKey, - chainID: chainID, + cachePrefix: cachePrefix, ServiceLogger: logger, } } @@ -73,7 +73,7 @@ func (c *ServiceCache) GetCachedQueryResponse( ctx context.Context, req *decode.EVMRPCRequestEnvelope, ) ([]byte, error) { - key, err := GetQueryKey(c.chainID, req) + key, err := GetQueryKey(c.cachePrefix, req) if err != nil { return nil, err } @@ -106,7 +106,7 @@ func (c *ServiceCache) CacheQueryResponse( return fmt.Errorf("response isn't cacheable") } - key, err := GetQueryKey(c.chainID, req) + key, err := GetQueryKey(c.cachePrefix, req) if err != nil { return err } diff --git a/service/cachemdw/cache_test.go b/service/cachemdw/cache_test.go index 626422a..3aef28e 100644 --- a/service/cachemdw/cache_test.go +++ b/service/cachemdw/cache_test.go @@ -18,14 +18,13 @@ import ( ) const ( - defaultChainIDString = "1" - defaultHost = "api.kava.io" - defaultBlockNumber = "42" + defaultCachePrefixString = "1" + defaultBlockNumber = "42" ) var ( - defaultChainID = big.NewInt(1) - defaultQueryResp = []byte(testEVMQueries[TestRequestWeb3ClientVersion].ResponseBody) + defaultCachePrefix = big.NewInt(1) + defaultQueryResp = []byte(testEVMQueries[TestRequestWeb3ClientVersion].ResponseBody) ) type MockEVMBlockGetter struct{} @@ -40,10 +39,6 @@ func (c *MockEVMBlockGetter) BlockByHash(ctx context.Context, hash common.Hash) panic("not implemented") } -func (c *MockEVMBlockGetter) ChainID(ctx context.Context) (*big.Int, error) { - return defaultChainID, nil -} - func TestUnitTestIsCacheable(t *testing.T) { logger, err := logging.New("TRACE") require.NoError(t, err) @@ -83,7 +78,7 @@ func TestUnitTestCacheQueryResponse(t *testing.T) { cacheTTL := time.Hour ctxb := context.Background() - serviceCache := cachemdw.NewServiceCache(inMemoryCache, blockGetter, cacheTTL, service.DecodedRequestContextKey, defaultChainIDString, &logger) + serviceCache := cachemdw.NewServiceCache(inMemoryCache, blockGetter, cacheTTL, service.DecodedRequestContextKey, defaultCachePrefixString, &logger) req := mkEVMRPCRequestEnvelope(defaultBlockNumber) resp, err := serviceCache.GetCachedQueryResponse(ctxb, req) diff --git a/service/cachemdw/keys.go b/service/cachemdw/keys.go index f3ab361..9a51b25 100644 --- a/service/cachemdw/keys.go +++ b/service/cachemdw/keys.go @@ -38,7 +38,7 @@ func BuildCacheKey(cacheItemType CacheItemType, parts []string) string { // GetQueryKey calculates cache key for request func GetQueryKey( - chainID string, + cachePrefix string, req *decode.EVMRPCRequestEnvelope, ) (string, error) { if req == nil { @@ -58,7 +58,7 @@ func GetQueryKey( hashedReq := crypto.Keccak256Hash(data) parts := []string{ - chainID, + cachePrefix, req.Method, hashedReq.Hex(), } diff --git a/service/cachemdw/keys_test.go b/service/cachemdw/keys_test.go index d67574f..06cdd01 100644 --- a/service/cachemdw/keys_test.go +++ b/service/cachemdw/keys_test.go @@ -33,14 +33,14 @@ func TestUnitTestBuildCacheKey(t *testing.T) { func TestUnitTestGetQueryKey(t *testing.T) { for _, tc := range []struct { desc string - chainID string + cachePrefix string req *decode.EVMRPCRequestEnvelope expectedCacheKey string errMsg string }{ { - desc: "test case #1", - chainID: "chain1", + desc: "test case #1", + cachePrefix: "chain1", req: &decode.EVMRPCRequestEnvelope{ JSONRPCVersion: "2.0", ID: 1, @@ -50,14 +50,14 @@ func TestUnitTestGetQueryKey(t *testing.T) { expectedCacheKey: "query:chain1:eth_getBlockByHash:0xb2b69f976d9aa41cd2065e2a2354254f6cba682a6fe2b3996571daa27ea4a6f4", }, { - desc: "test case #1", - chainID: "chain1", - req: nil, - errMsg: "request shouldn't be nil", + desc: "test case #1", + cachePrefix: "chain1", + req: nil, + errMsg: "request shouldn't be nil", }, } { t.Run(tc.desc, func(t *testing.T) { - cacheKey, err := cachemdw.GetQueryKey(tc.chainID, tc.req) + cacheKey, err := cachemdw.GetQueryKey(tc.cachePrefix, tc.req) if tc.errMsg == "" { require.NoError(t, err) require.Equal(t, tc.expectedCacheKey, cacheKey) diff --git a/service/cachemdw/middleware_test.go b/service/cachemdw/middleware_test.go index f3d589e..c08ad56 100644 --- a/service/cachemdw/middleware_test.go +++ b/service/cachemdw/middleware_test.go @@ -24,7 +24,7 @@ func TestE2ETestServiceCacheMiddleware(t *testing.T) { blockGetter := NewMockEVMBlockGetter() cacheTTL := time.Duration(0) // TTL: no expiry - serviceCache := cachemdw.NewServiceCache(inMemoryCache, blockGetter, cacheTTL, service.DecodedRequestContextKey, defaultChainIDString, &logger) + serviceCache := cachemdw.NewServiceCache(inMemoryCache, blockGetter, cacheTTL, service.DecodedRequestContextKey, defaultCachePrefixString, &logger) emptyHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}) cachingMdw := serviceCache.CachingMiddleware(emptyHandler) diff --git a/service/service.go b/service/service.go index 26637a9..a607c67 100644 --- a/service/service.go +++ b/service/service.go @@ -212,7 +212,7 @@ func createServiceCache( evmclient, config.CacheTTL, DecodedRequestContextKey, - config.ChainID, + config.CachePrefix, logger, ) From 952fdc43f4592d982c78ee0995d676a2a5fcb15c Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 18 Oct 2023 17:49:09 -0400 Subject: [PATCH 13/29] Change Key Structure --- config/config.go | 2 +- service/cachemdw/keys.go | 22 ++++++++++++---------- service/cachemdw/keys_test.go | 10 ++++++---- service/cachemdw/middleware_test.go | 2 +- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/config/config.go b/config/config.go index 2963d37..baa784a 100644 --- a/config/config.go +++ b/config/config.go @@ -42,7 +42,7 @@ type Config struct { // TTL for cached evm requests CacheTTL time.Duration // CachePrefix is used as prefix for any key in the cache, key has such structure: - // query::: + // :evm-request::sha256: // Possible values are testnet, mainnet, etc... CachePrefix string } diff --git a/service/cachemdw/keys.go b/service/cachemdw/keys.go index 9a51b25..fb83a0c 100644 --- a/service/cachemdw/keys.go +++ b/service/cachemdw/keys.go @@ -1,33 +1,34 @@ package cachemdw import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "strings" - "github.com/ethereum/go-ethereum/crypto" - "github.com/kava-labs/kava-proxy-service/decode" ) type CacheItemType int const ( - CacheItemTypeQuery CacheItemType = iota + 1 + CacheItemTypeEVMRequest CacheItemType = iota + 1 ) func (t CacheItemType) String() string { switch t { - case CacheItemTypeQuery: - return "query" + case CacheItemTypeEVMRequest: + return "evm-request" default: return "unknown" } } -func BuildCacheKey(cacheItemType CacheItemType, parts []string) string { +func BuildCacheKey(cachePrefix string, cacheItemType CacheItemType, parts []string) string { fullParts := append( []string{ + cachePrefix, cacheItemType.String(), }, parts..., @@ -55,13 +56,14 @@ func GetQueryKey( data = append(data, []byte(req.Method)...) data = append(data, serializedParams...) - hashedReq := crypto.Keccak256Hash(data) + hashedReq := sha256.Sum256(data) + hashedReqInHex := hex.EncodeToString(hashedReq[:]) parts := []string{ - cachePrefix, req.Method, - hashedReq.Hex(), + "sha256", + hashedReqInHex, } - return BuildCacheKey(CacheItemTypeQuery, parts), nil + return BuildCacheKey(cachePrefix, CacheItemTypeEVMRequest, parts), nil } diff --git a/service/cachemdw/keys_test.go b/service/cachemdw/keys_test.go index 06cdd01..703dd94 100644 --- a/service/cachemdw/keys_test.go +++ b/service/cachemdw/keys_test.go @@ -12,19 +12,21 @@ import ( func TestUnitTestBuildCacheKey(t *testing.T) { for _, tc := range []struct { desc string + cachePrefix string cacheItemType cachemdw.CacheItemType parts []string expectedCacheKey string }{ { desc: "test case #1", - cacheItemType: cachemdw.CacheItemTypeQuery, + cachePrefix: "chain1", + cacheItemType: cachemdw.CacheItemTypeEVMRequest, parts: []string{"1", "2", "3"}, - expectedCacheKey: "query:1:2:3", + expectedCacheKey: "chain1:evm-request:1:2:3", }, } { t.Run(tc.desc, func(t *testing.T) { - cacheKey := cachemdw.BuildCacheKey(tc.cacheItemType, tc.parts) + cacheKey := cachemdw.BuildCacheKey(tc.cachePrefix, tc.cacheItemType, tc.parts) require.Equal(t, tc.expectedCacheKey, cacheKey) }) } @@ -47,7 +49,7 @@ func TestUnitTestGetQueryKey(t *testing.T) { Method: "eth_getBlockByHash", Params: []interface{}{"0x1234", true}, }, - expectedCacheKey: "query:chain1:eth_getBlockByHash:0xb2b69f976d9aa41cd2065e2a2354254f6cba682a6fe2b3996571daa27ea4a6f4", + expectedCacheKey: "chain1:evm-request:eth_getBlockByHash:sha256:2db366278f2cb463f92147bd888bdcad528b44baa94b7920fdff35f4c11ee617", }, { desc: "test case #1", diff --git a/service/cachemdw/middleware_test.go b/service/cachemdw/middleware_test.go index c08ad56..03e8eb3 100644 --- a/service/cachemdw/middleware_test.go +++ b/service/cachemdw/middleware_test.go @@ -65,7 +65,7 @@ func TestE2ETestServiceCacheMiddleware(t *testing.T) { cacheItems := inMemoryCache.GetAll(context.Background()) require.Len(t, cacheItems, 1) - require.Contains(t, cacheItems, "query:1:eth_getBlockByNumber:0x885d3d84b42d647be47d94a001428be7e88ab787251031ddbfb247a581d0505a") + require.Contains(t, cacheItems, "1:evm-request:eth_getBlockByNumber:sha256:bf79de57723b25b85391513b470ea6989e7c44dd9afc0c270ee961c9f12f578d") }) t.Run("cache hit", func(t *testing.T) { From e1ed01e6ad8efc9e5648669e744c2500fe2b32cd Mon Sep 17 00:00:00 2001 From: Evgeniy Scherbina Date: Wed, 18 Oct 2023 17:52:41 -0400 Subject: [PATCH 14/29] Update CACHING.md --- architecture/CACHING.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/architecture/CACHING.md b/architecture/CACHING.md index 355ebb2..421e138 100644 --- a/architecture/CACHING.md +++ b/architecture/CACHING.md @@ -55,31 +55,31 @@ we don't cache requests without `specific block number` or requests which uses m Keys have such format: -`query:::` +`:evm-request::sha256:` For example: -`query:local-chain:eth_getBlockByNumber:0x72806e50da4f1c824b9d5a74ce9d76ac4db72e4da049802d1d6f2de3fda73e10` +`local-chain:evm-request:eth_getBlockByHash:sha256:2db366278f2cb463f92147bd888bdcad528b44baa94b7920fdff35f4c11ee617` ### Invalidation for specific method If you want to invalidate cache for specific method you may run such command: -`redis-cli KEYS "query:::*" | xargs redis-cli DEL` +`redis-cli KEYS ":evm-request::sha256:*" | xargs redis-cli DEL` For example: -`redis-cli KEYS "query:local-chain:eth_getBlockByNumber:*" | xargs redis-cli DEL` +`redis-cli KEYS "local-chain:evm-request:eth_getBlockByNumber:sha256:*" | xargs redis-cli DEL` ### Invalidation for all methods If you want to invalidate cache for all methods you may run such command: -`redis-cli KEYS "query::*" | xargs redis-cli DEL` +`redis-cli KEYS ":evm-request:*" | xargs redis-cli DEL` For example: -`redis-cli KEYS "query:local-chain:*" | xargs redis-cli DEL` +`redis-cli KEYS "local-chain:evm-request:*" | xargs redis-cli DEL` ## Architecture Diagrams From 667fa36d4c75056df7afa436e10f1c95a4ababd9 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Wed, 18 Oct 2023 18:04:21 -0400 Subject: [PATCH 15/29] Rename ChainID -> CachePrefix and add validation --- .env | 10 +++++++++- config/config.go | 15 +++++---------- config/validate.go | 14 ++++++++++++++ service/cachemdw/cache.go | 5 +++-- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/.env b/.env index a54ad67..fc6c8e2 100644 --- a/.env +++ b/.env @@ -89,9 +89,17 @@ METRIC_PARTITIONING_ROUTINE_DELAY_FIRST_RUN_SECONDS=10 METRIC_PARTITIONINING_PREFILL_PERIOD_DAYS=7 # Used by `ready` script to ensure metric partitions have been created. MINIMUM_REQUIRED_PARTITIONS=30 +# RedisEndpointURL is an url of redis REDIS_ENDPOINT_URL=redis:6379 REDIS_PASSWORD= -CHAIN_ID=local-chain +# TTL for cached evm requests +# TTL should be specified in seconds +CACHE_TTL=600 +# CachePrefix is used as prefix for any key in the cache, key has such structure: +# :evm-request::sha256: +# Possible values are testnet, mainnet, etc... +# CachePrefix must not contain colon symbol +CACHE_PREFIX=local-chain ##### Database Config POSTGRES_PASSWORD=password diff --git a/config/config.go b/config/config.go index baa784a..256a54a 100644 --- a/config/config.go +++ b/config/config.go @@ -39,12 +39,8 @@ type Config struct { MetricPartitioningPrefillPeriodDays int RedisEndpointURL string RedisPassword string - // TTL for cached evm requests - CacheTTL time.Duration - // CachePrefix is used as prefix for any key in the cache, key has such structure: - // :evm-request::sha256: - // Possible values are testnet, mainnet, etc... - CachePrefix string + CacheTTL time.Duration + CachePrefix string } const ( @@ -90,8 +86,7 @@ const ( REDIS_ENDPOINT_URL_ENVIRONMENT_KEY = "REDIS_ENDPOINT_URL" REDIS_PASSWORD_ENVIRONMENT_KEY = "REDIS_PASSWORD" CACHE_TTL_ENVIRONMENT_KEY = "CACHE_TTL" - DEFAULT_CACHE_TTL_SECONDS = 600 - CHAIN_ID_ENVIRONMENT_KEY = "CHAIN_ID" + CACHE_PREFIX_ENVIRONMENT_KEY = "CACHE_PREFIX" ) // EnvOrDefault fetches an environment variable value, or if not set returns the fallback value @@ -219,7 +214,7 @@ func ReadConfig() Config { MetricPartitioningPrefillPeriodDays: EnvOrDefaultInt(METRIC_PARTITIONING_PREFILL_PERIOD_DAYS_ENVIRONMENT_KEY, DEFAULT_METRIC_PARTITIONING_PREFILL_PERIOD_DAYS), RedisEndpointURL: os.Getenv(REDIS_ENDPOINT_URL_ENVIRONMENT_KEY), RedisPassword: os.Getenv(REDIS_PASSWORD_ENVIRONMENT_KEY), - CacheTTL: time.Duration(EnvOrDefaultInt(CACHE_TTL_ENVIRONMENT_KEY, DEFAULT_CACHE_TTL_SECONDS)) * time.Second, - CachePrefix: os.Getenv(CHAIN_ID_ENVIRONMENT_KEY), + CacheTTL: time.Duration(EnvOrDefaultInt(CACHE_TTL_ENVIRONMENT_KEY, 0)) * time.Second, + CachePrefix: os.Getenv(CACHE_PREFIX_ENVIRONMENT_KEY), } } diff --git a/config/validate.go b/config/validate.go index 446fb1c..afa15c8 100644 --- a/config/validate.go +++ b/config/validate.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "strconv" + "strings" ) var ( @@ -48,5 +49,18 @@ func Validate(config Config) error { allErrs = errors.Join(allErrs, fmt.Errorf("invalid %s specified %d, must be non-zero and less than or equal to %d", METRIC_PARTITIONING_PREFILL_PERIOD_DAYS_ENVIRONMENT_KEY, config.MetricPartitioningPrefillPeriodDays, MaxMetricPartitioningPrefillPeriodDays)) } + if config.RedisEndpointURL == "" { + allErrs = errors.Join(allErrs, fmt.Errorf("invalid %s specified %s, must not be empty", REDIS_ENDPOINT_URL_ENVIRONMENT_KEY, config.RedisEndpointURL)) + } + if config.CacheTTL <= 0 { + allErrs = errors.Join(allErrs, fmt.Errorf("invalid %s specified %s, must be greater than zero", CACHE_TTL_ENVIRONMENT_KEY, config.CacheTTL)) + } + if strings.Contains(config.CachePrefix, ":") { + allErrs = errors.Join(allErrs, fmt.Errorf("invalid %s specified %s, must not contain colon symbol", CACHE_PREFIX_ENVIRONMENT_KEY, config.CachePrefix)) + } + if config.CachePrefix == "" { + allErrs = errors.Join(allErrs, fmt.Errorf("invalid %s specified %s, must not be empty", CACHE_PREFIX_ENVIRONMENT_KEY, config.CachePrefix)) + } + return allErrs } diff --git a/service/cachemdw/cache.go b/service/cachemdw/cache.go index 7ecaa11..8114420 100644 --- a/service/cachemdw/cache.go +++ b/service/cachemdw/cache.go @@ -14,8 +14,9 @@ import ( // ServiceCache is responsible for caching EVM requests and provides corresponding middleware // ServiceCache can work with any underlying storage which implements simple cache.Cache interface type ServiceCache struct { - cacheClient cache.Cache - blockGetter decode.EVMBlockGetter + cacheClient cache.Cache + blockGetter decode.EVMBlockGetter + // TTL for cached evm requests cacheTTL time.Duration decodedRequestContextKey any // cachePrefix is used as prefix for any key in the cache From 9ff12c4bd250ca348330b7f5898584fdf3c00f9d Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 19 Oct 2023 11:50:08 -0400 Subject: [PATCH 16/29] Add is_cache_enabled boolean flag --- .env | 12 +++++---- config/config.go | 3 +++ main_test.go | 8 +++--- service/cachemdw/cache.go | 5 +++- service/cachemdw/cache_test.go | 10 +++++++- service/cachemdw/caching_middleware.go | 6 +++++ service/cachemdw/is_cached_middleware.go | 6 +++++ service/cachemdw/middleware_test.go | 19 +++++++++++++-- service/middleware.go | 31 +++++++++++++----------- service/service.go | 1 + 10 files changed, 74 insertions(+), 27 deletions(-) diff --git a/.env b/.env index fc6c8e2..2474685 100644 --- a/.env +++ b/.env @@ -89,16 +89,18 @@ METRIC_PARTITIONING_ROUTINE_DELAY_FIRST_RUN_SECONDS=10 METRIC_PARTITIONINING_PREFILL_PERIOD_DAYS=7 # Used by `ready` script to ensure metric partitions have been created. MINIMUM_REQUIRED_PARTITIONS=30 -# RedisEndpointURL is an url of redis +# CACHE_ENABLED specifies if cache should be enabled. By default cache is disabled. +CACHE_ENABLED=true +# REDIS_ENDPOINT_URL is an url of redis REDIS_ENDPOINT_URL=redis:6379 REDIS_PASSWORD= -# TTL for cached evm requests -# TTL should be specified in seconds +# CACHE_TTL is a TTL for cached evm requests +# CACHE_TTL should be specified in seconds CACHE_TTL=600 -# CachePrefix is used as prefix for any key in the cache, key has such structure: +# CACHE_PREFIX is used as prefix for any key in the cache, key has such structure: # :evm-request::sha256: # Possible values are testnet, mainnet, etc... -# CachePrefix must not contain colon symbol +# CACHE_PREFIX must not contain colon symbol CACHE_PREFIX=local-chain ##### Database Config diff --git a/config/config.go b/config/config.go index 256a54a..8260ef3 100644 --- a/config/config.go +++ b/config/config.go @@ -37,6 +37,7 @@ type Config struct { MetricPartitioningRoutineInterval time.Duration MetricPartitioningRoutineDelayFirstRun time.Duration MetricPartitioningPrefillPeriodDays int + CacheEnabled bool RedisEndpointURL string RedisPassword string CacheTTL time.Duration @@ -83,6 +84,7 @@ const ( DEFAULT_DATABASE_READ_TIMEOUT_SECONDS = 60 DATABASE_WRITE_TIMEOUT_SECONDS_ENVIRONMENT_KEY = "DATABASE_WRITE_TIMEOUT_SECONDS" DEFAULT_DATABASE_WRITE_TIMEOUT_SECONDS = 10 + CACHE_ENABLED_ENVIRONMENT_KEY = "CACHE_ENABLED" REDIS_ENDPOINT_URL_ENVIRONMENT_KEY = "REDIS_ENDPOINT_URL" REDIS_PASSWORD_ENVIRONMENT_KEY = "REDIS_PASSWORD" CACHE_TTL_ENVIRONMENT_KEY = "CACHE_TTL" @@ -212,6 +214,7 @@ func ReadConfig() Config { MetricPartitioningRoutineInterval: time.Duration(time.Duration(EnvOrDefaultInt(METRIC_PARTITIONING_ROUTINE_INTERVAL_SECONDS_ENVIRONMENT_KEY, DEFAULT_METRIC_PARTITIONING_ROUTINE_INTERVAL_SECONDS)) * time.Second), MetricPartitioningRoutineDelayFirstRun: time.Duration(time.Duration(EnvOrDefaultInt(METRIC_PARTITIONING_ROUTINE_DELAY_FIRST_RUN_SECONDS_ENVIRONMENT_KEY, DEFAULT_METRIC_PARTITIONING_ROUTINE_DELAY_FIRST_RUN_SECONDS)) * time.Second), MetricPartitioningPrefillPeriodDays: EnvOrDefaultInt(METRIC_PARTITIONING_PREFILL_PERIOD_DAYS_ENVIRONMENT_KEY, DEFAULT_METRIC_PARTITIONING_PREFILL_PERIOD_DAYS), + CacheEnabled: EnvOrDefaultBool(CACHE_ENABLED_ENVIRONMENT_KEY, false), RedisEndpointURL: os.Getenv(REDIS_ENDPOINT_URL_ENVIRONMENT_KEY), RedisPassword: os.Getenv(REDIS_PASSWORD_ENVIRONMENT_KEY), CacheTTL: time.Duration(EnvOrDefaultInt(CACHE_TTL_ENVIRONMENT_KEY, 0)) * time.Second, diff --git a/main_test.go b/main_test.go index 40a96c3..9415fbc 100644 --- a/main_test.go +++ b/main_test.go @@ -483,7 +483,7 @@ func TestE2ETestProxyTracksBlockNumberForMethodsWithBlockHashParam(t *testing.T) } } -func TestE2eTestCachingMdwWithBlockNumberParam(t *testing.T) { +func TestE2ETestCachingMdwWithBlockNumberParam(t *testing.T) { // create api and database clients client, err := ethclient.Dial(proxyServiceURL) if err != nil { @@ -513,7 +513,7 @@ func TestE2eTestCachingMdwWithBlockNumberParam(t *testing.T) { } { t.Run(tc.desc, func(t *testing.T) { // test cache MISS and cache HIT scenarios for specified method - // check corresponding values in cachemdw.CacheMissHeaderValue HTTP header + // check corresponding values in cachemdw.CacheHeaderKey HTTP header // check that cached and non-cached responses are equal // eth_getBlockByNumber - cache MISS @@ -557,7 +557,7 @@ func TestE2eTestCachingMdwWithBlockNumberParam(t *testing.T) { cleanUpRedis(t, redisClient) } -func TestE2eTestCachingMdwWithBlockNumberParam_EmptyResult(t *testing.T) { +func TestE2ETestCachingMdwWithBlockNumberParam_EmptyResult(t *testing.T) { testRandomAddressHex := "0x6767114FFAA17C6439D7AEA480738B982CE63A02" testAddress := common.HexToAddress(testRandomAddressHex) @@ -590,7 +590,7 @@ func TestE2eTestCachingMdwWithBlockNumberParam_EmptyResult(t *testing.T) { } { t.Run(tc.desc, func(t *testing.T) { // both calls should lead to cache MISS scenario, because empty results aren't cached - // check corresponding values in cachemdw.CacheMissHeaderValue HTTP header + // check corresponding values in cachemdw.CacheHeaderKey HTTP header // check that responses are equal // eth_getBlockByNumber - cache MISS diff --git a/service/cachemdw/cache.go b/service/cachemdw/cache.go index 8114420..9dd8f57 100644 --- a/service/cachemdw/cache.go +++ b/service/cachemdw/cache.go @@ -20,7 +20,8 @@ type ServiceCache struct { cacheTTL time.Duration decodedRequestContextKey any // cachePrefix is used as prefix for any key in the cache - cachePrefix string + cachePrefix string + cacheEnabled bool *logging.ServiceLogger } @@ -31,6 +32,7 @@ func NewServiceCache( cacheTTL time.Duration, decodedRequestContextKey any, cachePrefix string, + cacheEnabled bool, logger *logging.ServiceLogger, ) *ServiceCache { return &ServiceCache{ @@ -39,6 +41,7 @@ func NewServiceCache( cacheTTL: cacheTTL, decodedRequestContextKey: decodedRequestContextKey, cachePrefix: cachePrefix, + cacheEnabled: cacheEnabled, ServiceLogger: logger, } } diff --git a/service/cachemdw/cache_test.go b/service/cachemdw/cache_test.go index 3aef28e..7121a32 100644 --- a/service/cachemdw/cache_test.go +++ b/service/cachemdw/cache_test.go @@ -78,7 +78,15 @@ func TestUnitTestCacheQueryResponse(t *testing.T) { cacheTTL := time.Hour ctxb := context.Background() - serviceCache := cachemdw.NewServiceCache(inMemoryCache, blockGetter, cacheTTL, service.DecodedRequestContextKey, defaultCachePrefixString, &logger) + serviceCache := cachemdw.NewServiceCache( + inMemoryCache, + blockGetter, + cacheTTL, + service.DecodedRequestContextKey, + defaultCachePrefixString, + true, + &logger, + ) req := mkEVMRPCRequestEnvelope(defaultBlockNumber) resp, err := serviceCache.GetCachedQueryResponse(ctxb, req) diff --git a/service/cachemdw/caching_middleware.go b/service/cachemdw/caching_middleware.go index ec41d9f..547b3a1 100644 --- a/service/cachemdw/caching_middleware.go +++ b/service/cachemdw/caching_middleware.go @@ -19,6 +19,12 @@ func (c *ServiceCache) CachingMiddleware( next http.Handler, ) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + // if cache is not enabled - do nothing and forward to next middleware + if !c.cacheEnabled { + next.ServeHTTP(w, r) + return + } + // if we can't get decoded request then forward to next middleware req := r.Context().Value(c.decodedRequestContextKey) decodedReq, ok := (req).(*decode.EVMRPCRequestEnvelope) diff --git a/service/cachemdw/is_cached_middleware.go b/service/cachemdw/is_cached_middleware.go index 3cbc557..6936556 100644 --- a/service/cachemdw/is_cached_middleware.go +++ b/service/cachemdw/is_cached_middleware.go @@ -28,6 +28,12 @@ func (c *ServiceCache) IsCachedMiddleware( next http.Handler, ) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + // if cache is not enabled - do nothing and forward to next middleware + if !c.cacheEnabled { + next.ServeHTTP(w, r) + return + } + uncachedContext := context.WithValue(r.Context(), CachedContextKey, false) cachedContext := context.WithValue(r.Context(), CachedContextKey, true) diff --git a/service/cachemdw/middleware_test.go b/service/cachemdw/middleware_test.go index 03e8eb3..a819a50 100644 --- a/service/cachemdw/middleware_test.go +++ b/service/cachemdw/middleware_test.go @@ -16,7 +16,7 @@ import ( "github.com/kava-labs/kava-proxy-service/service/cachemdw" ) -func TestE2ETestServiceCacheMiddleware(t *testing.T) { +func TestUnitTestServiceCacheMiddleware(t *testing.T) { logger, err := logging.New("TRACE") require.NoError(t, err) @@ -24,7 +24,15 @@ func TestE2ETestServiceCacheMiddleware(t *testing.T) { blockGetter := NewMockEVMBlockGetter() cacheTTL := time.Duration(0) // TTL: no expiry - serviceCache := cachemdw.NewServiceCache(inMemoryCache, blockGetter, cacheTTL, service.DecodedRequestContextKey, defaultCachePrefixString, &logger) + serviceCache := cachemdw.NewServiceCache( + inMemoryCache, + blockGetter, + cacheTTL, + service.DecodedRequestContextKey, + defaultCachePrefixString, + true, + &logger, + ) emptyHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}) cachingMdw := serviceCache.CachingMiddleware(emptyHandler) @@ -49,6 +57,9 @@ func TestE2ETestServiceCacheMiddleware(t *testing.T) { }) isCachedMdw := serviceCache.IsCachedMiddleware(proxyHandler) + // test cache MISS and cache HIT scenarios for specified method + // check corresponding values in cachemdw.CacheHeaderKey HTTP header + t.Run("cache miss", func(t *testing.T) { req := createTestHttpRequest( t, @@ -81,6 +92,10 @@ func TestE2ETestServiceCacheMiddleware(t *testing.T) { require.Equal(t, http.StatusOK, resp.Code) require.JSONEq(t, testEVMQueries[TestRequestEthBlockByNumberSpecific].ResponseBody, resp.Body.String()) require.Equal(t, cachemdw.CacheHitHeaderValue, resp.Header().Get(cachemdw.CacheHeaderKey)) + + cacheItems := inMemoryCache.GetAll(context.Background()) + require.Len(t, cacheItems, 1) + require.Contains(t, cacheItems, "1:evm-request:eth_getBlockByNumber:sha256:bf79de57723b25b85391513b470ea6989e7c44dd9afc0c270ee961c9f12f578d") }) } diff --git a/service/middleware.go b/service/middleware.go index 82ae226..f332b06 100644 --- a/service/middleware.go +++ b/service/middleware.go @@ -233,9 +233,9 @@ func createProxyRequestMiddleware(next http.Handler, config config.Config, servi response := r.Context().Value(cachemdw.ResponseContextKey) typedResponse, ok := response.([]byte) - // if request is cached and response is present in context - serve the request from the cache + // if cache is enabled, request is cached and response is present in context - serve the request from the cache // otherwise proxy to the actual backend - if isCached && ok { + if config.CacheEnabled && isCached && ok { serviceLogger.Logger.Trace(). Str("method", r.Method). Str("url", r.URL.String()). @@ -271,20 +271,23 @@ func createProxyRequestMiddleware(next http.Handler, config config.Config, servi // extract the original hostname the request was sent to requestHostnameContext := context.WithValue(originRoundtripLatencyContext, RequestHostnameContextKey, r.Host) - var bodyCopy bytes.Buffer - tee := io.TeeReader(lrw.body, &bodyCopy) - // read all body from reader into bodyBytes, and copy into bodyCopy - bodyBytes, err := io.ReadAll(tee) - if err != nil { - serviceLogger.Error().Err(err).Msg("can't read lrw.body") - } + enrichedContext := requestHostnameContext - // replace empty body reader with fresh copy - lrw.body = &bodyCopy - // set body in context - responseContext := context.WithValue(requestHostnameContext, cachemdw.ResponseContextKey, bodyBytes) + // if cache is enabled, update enrichedContext with cachemdw.ResponseContextKey -> bodyBytes key-value pair + if config.CacheEnabled { + var bodyCopy bytes.Buffer + tee := io.TeeReader(lrw.body, &bodyCopy) + // read all body from reader into bodyBytes, and copy into bodyCopy + bodyBytes, err := io.ReadAll(tee) + if err != nil { + serviceLogger.Error().Err(err).Msg("can't read lrw.body") + } - enrichedContext := responseContext + // replace empty body reader with fresh copy + lrw.body = &bodyCopy + // set body in context + enrichedContext = context.WithValue(enrichedContext, cachemdw.ResponseContextKey, bodyBytes) + } // parse the remote address of the request for use below remoteAddressParts := strings.Split(r.RemoteAddr, ":") diff --git a/service/service.go b/service/service.go index a607c67..dbefc22 100644 --- a/service/service.go +++ b/service/service.go @@ -213,6 +213,7 @@ func createServiceCache( config.CacheTTL, DecodedRequestContextKey, config.CachePrefix, + config.CacheEnabled, logger, ) From 3b2b122494f546e1820c21bcf941953b8e6bfe8f Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 19 Oct 2023 14:00:09 -0400 Subject: [PATCH 17/29] Added test for cache disabled scenario --- service/cachemdw/middleware_test.go | 81 +++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/service/cachemdw/middleware_test.go b/service/cachemdw/middleware_test.go index a819a50..8476e8c 100644 --- a/service/cachemdw/middleware_test.go +++ b/service/cachemdw/middleware_test.go @@ -99,6 +99,87 @@ func TestUnitTestServiceCacheMiddleware(t *testing.T) { }) } +func TestUnitTestServiceCacheMiddleware_CacheIsDisabled(t *testing.T) { + logger, err := logging.New("TRACE") + require.NoError(t, err) + + inMemoryCache := cache.NewInMemoryCache() + blockGetter := NewMockEVMBlockGetter() + cacheTTL := time.Duration(0) // TTL: no expiry + + serviceCache := cachemdw.NewServiceCache( + inMemoryCache, + blockGetter, + cacheTTL, + service.DecodedRequestContextKey, + defaultCachePrefixString, + false, + &logger, + ) + + emptyHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}) + cachingMdw := serviceCache.CachingMiddleware(emptyHandler) + // proxyHandler emulates behaviour of actual service proxy handler + // sequence of execution: + // - isCachedMdw + // - proxyHandler + // - cachingMdw + // - emptyHandler + proxyHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + response := []byte(testEVMQueries[TestRequestEthBlockByNumberSpecific].ResponseBody) + if cachemdw.IsRequestCached(r.Context()) { + w.Header().Add(cachemdw.CacheHeaderKey, cachemdw.CacheHitHeaderValue) + } else { + w.Header().Add(cachemdw.CacheHeaderKey, cachemdw.CacheMissHeaderValue) + } + w.WriteHeader(http.StatusOK) + w.Write(response) + responseContext := context.WithValue(r.Context(), cachemdw.ResponseContextKey, response) + + cachingMdw.ServeHTTP(w, r.WithContext(responseContext)) + }) + isCachedMdw := serviceCache.IsCachedMiddleware(proxyHandler) + + // both calls should lead to cache MISS scenario, because cache is disabled + // check corresponding values in cachemdw.CacheHeaderKey HTTP header + + t.Run("cache miss", func(t *testing.T) { + req := createTestHttpRequest( + t, + "https://api.kava.io:8545/thisshouldntshowup", + TestRequestEthBlockByNumberSpecific, + ) + resp := httptest.NewRecorder() + + isCachedMdw.ServeHTTP(resp, req) + + require.Equal(t, http.StatusOK, resp.Code) + require.JSONEq(t, testEVMQueries[TestRequestEthBlockByNumberSpecific].ResponseBody, resp.Body.String()) + require.Equal(t, cachemdw.CacheMissHeaderValue, resp.Header().Get(cachemdw.CacheHeaderKey)) + + cacheItems := inMemoryCache.GetAll(context.Background()) + require.Len(t, cacheItems, 0) + }) + + t.Run("cache miss again (cache is disabled)", func(t *testing.T) { + req := createTestHttpRequest( + t, + "https://api.kava.io:8545/thisshouldntshowup", + TestRequestEthBlockByNumberSpecific, + ) + resp := httptest.NewRecorder() + + isCachedMdw.ServeHTTP(resp, req) + + require.Equal(t, http.StatusOK, resp.Code) + require.JSONEq(t, testEVMQueries[TestRequestEthBlockByNumberSpecific].ResponseBody, resp.Body.String()) + require.Equal(t, cachemdw.CacheMissHeaderValue, resp.Header().Get(cachemdw.CacheHeaderKey)) + + cacheItems := inMemoryCache.GetAll(context.Background()) + require.Len(t, cacheItems, 0) + }) +} + func createTestHttpRequest( t *testing.T, url string, From 8416e195552903ca5993214e698c103f0912011c Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 19 Oct 2023 14:40:20 -0400 Subject: [PATCH 18/29] CR's fixes --- .env | 2 ++ main_test.go | 7 +++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 2474685..7332adc 100644 --- a/.env +++ b/.env @@ -41,6 +41,8 @@ TEST_SERVICE_LOG_LEVEL=ERROR # endpoint the proxy service should use for querying # evm blockchain information related to proxied requests TEST_EVM_QUERY_SERVICE_URL=http://kava:8545 +# TEST_REDIS_ENDPOINT_URL is an url of redis +TEST_REDIS_ENDPOINT_URL=localhost:6379 ##### Kava Node Config diff --git a/main_test.go b/main_test.go index 9415fbc..3d08fa8 100644 --- a/main_test.go +++ b/main_test.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "io" "math/big" "net/http" @@ -61,7 +60,7 @@ var ( RunDatabaseMigrations: false, } - redisHostPort = os.Getenv("REDIS_HOST_PORT") + redisURL = os.Getenv("TEST_REDIS_ENDPOINT_URL") redisPassword = os.Getenv("REDIS_PASSWORD") ) @@ -491,7 +490,7 @@ func TestE2ETestCachingMdwWithBlockNumberParam(t *testing.T) { } redisClient := redis.NewClient(&redis.Options{ - Addr: fmt.Sprintf("localhost:%v", redisHostPort), + Addr: redisURL, Password: redisPassword, DB: 0, }) @@ -568,7 +567,7 @@ func TestE2ETestCachingMdwWithBlockNumberParam_EmptyResult(t *testing.T) { } redisClient := redis.NewClient(&redis.Options{ - Addr: fmt.Sprintf("localhost:%v", redisHostPort), + Addr: redisURL, Password: redisPassword, DB: 0, }) From 496113c34330bd35b36cbd84b57d56e68ace8be4 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 19 Oct 2023 15:36:01 -0400 Subject: [PATCH 19/29] Improve tests --- main_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/main_test.go b/main_test.go index 3d08fa8..1aed46c 100644 --- a/main_test.go +++ b/main_test.go @@ -523,6 +523,8 @@ func TestE2ETestCachingMdwWithBlockNumberParam(t *testing.T) { err = checkJsonRpcErr(body1) require.NoError(t, err) expectKeysNum(t, redisClient, tc.keysNum) + expectedKey := "local-chain:evm-request:eth_getBlockByNumber:sha256:d08b426164eacf6646fb1817403ec0af5d37869a0f32a01ebfab3096fa4999be" + containsKey(t, redisClient, expectedKey) // eth_getBlockByNumber - cache HIT resp2 := mkJsonRpcRequest(t, proxyServiceURL, tc.method, tc.params) @@ -532,6 +534,7 @@ func TestE2ETestCachingMdwWithBlockNumberParam(t *testing.T) { err = checkJsonRpcErr(body2) require.NoError(t, err) expectKeysNum(t, redisClient, tc.keysNum) + containsKey(t, redisClient, expectedKey) require.JSONEq(t, string(body1), string(body2), "blocks should be the same") }) @@ -544,11 +547,14 @@ func TestE2ETestCachingMdwWithBlockNumberParam(t *testing.T) { block1, err := client.BlockByNumber(testContext, big.NewInt(2)) require.NoError(t, err) expectKeysNum(t, redisClient, 2) + expectedKey := "local-chain:evm-request:eth_getBlockByNumber:sha256:0bfa7c5affc525ed731803c223042b4b1eb16ee7a6a539ae213b47a3ef6e3a7d" + containsKey(t, redisClient, expectedKey) // eth_getBlockByNumber - cache HIT block2, err := client.BlockByNumber(testContext, big.NewInt(2)) require.NoError(t, err) expectKeysNum(t, redisClient, 2) + containsKey(t, redisClient, expectedKey) require.Equal(t, block1, block2, "blocks should be the same") } @@ -640,6 +646,12 @@ func expectKeysNum(t *testing.T, redisClient *redis.Client, keysNum int) { require.Equal(t, keysNum, len(keys)) } +func containsKey(t *testing.T, redisClient *redis.Client, key string) { + keys, err := redisClient.Keys(context.Background(), key).Result() + require.NoError(t, err) + require.GreaterOrEqual(t, len(keys), 1) +} + func cleanUpRedis(t *testing.T, redisClient *redis.Client) { keys, err := redisClient.Keys(context.Background(), "*").Result() require.NoError(t, err) From 4706a07edd5b6d65b88db2a45eea31f374a24158 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 19 Oct 2023 16:07:26 -0400 Subject: [PATCH 20/29] Update healthcheck --- service/cachemdw/cache.go | 4 ++++ service/handlers.go | 20 +++++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/service/cachemdw/cache.go b/service/cachemdw/cache.go index 9dd8f57..0f2874e 100644 --- a/service/cachemdw/cache.go +++ b/service/cachemdw/cache.go @@ -121,3 +121,7 @@ func (c *ServiceCache) CacheQueryResponse( func (c *ServiceCache) Healthcheck(ctx context.Context) error { return c.cacheClient.Healthcheck(ctx) } + +func (c *ServiceCache) IsCacheEnabled() bool { + return c.cacheEnabled +} diff --git a/service/handlers.go b/service/handlers.go index da2fa3c..b7aa38c 100644 --- a/service/handlers.go +++ b/service/handlers.go @@ -26,15 +26,17 @@ func createHealthcheckHandler(service *ProxyService) func(http.ResponseWriter, * combinedErrors = errors.Join(combinedErrors, errMsg) } - // check that the cache is reachable - err = service.Cache.Healthcheck(context.Background()) - if err != nil { - service.Logger.Error(). - Err(err). - Msg("cache healthcheck failed") - - errMsg := fmt.Errorf("proxy service unable to connect to cache") - combinedErrors = errors.Join(combinedErrors, errMsg) + if service.Cache.IsCacheEnabled() { + // check that the cache is reachable + err := service.Cache.Healthcheck(context.Background()) + if err != nil { + service.Logger.Error(). + Err(err). + Msg("cache healthcheck failed") + + errMsg := fmt.Errorf("proxy service unable to connect to cache: %v", err) + combinedErrors = errors.Join(combinedErrors, errMsg) + } } if combinedErrors != nil { From 00581b0f23d24af0f1faf913b7053a884979238e Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 19 Oct 2023 16:15:21 -0400 Subject: [PATCH 21/29] CR's fixes --- service/middleware.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/service/middleware.go b/service/middleware.go index f332b06..0ce23e5 100644 --- a/service/middleware.go +++ b/service/middleware.go @@ -230,8 +230,8 @@ func createProxyRequestMiddleware(next http.Handler, config config.Config, servi } isCached := cachemdw.IsRequestCached(r.Context()) - response := r.Context().Value(cachemdw.ResponseContextKey) - typedResponse, ok := response.([]byte) + cachedResponse := r.Context().Value(cachemdw.ResponseContextKey) + typedCachedResponse, ok := cachedResponse.([]byte) // if cache is enabled, request is cached and response is present in context - serve the request from the cache // otherwise proxy to the actual backend @@ -244,7 +244,7 @@ func createProxyRequestMiddleware(next http.Handler, config config.Config, servi w.Header().Add(cachemdw.CacheHeaderKey, cachemdw.CacheHitHeaderValue) w.Header().Add("Content-Type", "application/json") - _, err := w.Write(typedResponse) + _, err := w.Write(typedCachedResponse) if err != nil { serviceLogger.Logger.Error().Msg(fmt.Sprintf("can't write cached response: %v", err)) } From 853b7266998ac1395a020030550655498b73b45e Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 19 Oct 2023 16:34:58 -0400 Subject: [PATCH 22/29] CR's fixes: improve logging --- service/middleware.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/service/middleware.go b/service/middleware.go index 0ce23e5..5d03ffa 100644 --- a/service/middleware.go +++ b/service/middleware.go @@ -162,6 +162,19 @@ func createProxyRequestMiddleware(next http.Handler, config config.Config, servi handler := func(proxies map[string]*httputil.ReverseProxy) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { + req := r.Context().Value(DecodedRequestContextKey) + decodedReq, ok := (req).(*decode.EVMRPCRequestEnvelope) + if !ok { + serviceLogger.Logger.Error(). + Str("method", r.Method). + Str("url", r.URL.String()). + Str("host", r.Host). + Msg("can't cast request to *EVMRPCRequestEnvelope type") + + // if we can't get decoded request then assign it empty structure to avoid panics + decodedReq = new(decode.EVMRPCRequestEnvelope) + } + serviceLogger.Trace().Msg(fmt.Sprintf("proxying request %+v", r)) proxyRequestAt := time.Now() @@ -240,6 +253,7 @@ func createProxyRequestMiddleware(next http.Handler, config config.Config, servi Str("method", r.Method). Str("url", r.URL.String()). Str("host", r.Host). + Str("evm-method", decodedReq.Method). Msg("cache hit") w.Header().Add(cachemdw.CacheHeaderKey, cachemdw.CacheHitHeaderValue) @@ -253,6 +267,7 @@ func createProxyRequestMiddleware(next http.Handler, config config.Config, servi Str("method", r.Method). Str("url", r.URL.String()). Str("host", r.Host). + Str("evm-method", decodedReq.Method). Msg("cache miss") w.Header().Add(cachemdw.CacheHeaderKey, cachemdw.CacheMissHeaderValue) From 50e72d8d1c6471f6296b395618984187b3db29cb Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 19 Oct 2023 17:45:01 -0400 Subject: [PATCH 23/29] Fixes after merge --- decode/evm_rpc.go | 7 ++----- service/cachemdw/cache_test.go | 4 ---- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/decode/evm_rpc.go b/decode/evm_rpc.go index 934ae83..5b97d14 100644 --- a/decode/evm_rpc.go +++ b/decode/evm_rpc.go @@ -11,12 +11,9 @@ import ( ethctypes "github.com/ethereum/go-ethereum/core/types" ) -// EVMBlockGetter defines an interface which can be implemented by any client capable of getting ethereum block by hash +// EVMBlockGetter defines an interface which can be implemented by any client capable of getting ethereum block header by hash type EVMBlockGetter interface { - // BlockByHash returns ethereum block by hash - BlockByHash(ctx context.Context, hash common.Hash) (*ethctypes.Block, error) - - // TODO(yevhenii): remote BlockByHash + AddComment + // HeaderByHash returns ethereum block header by hash HeaderByHash(ctx context.Context, hash common.Hash) (*ethctypes.Header, error) } diff --git a/service/cachemdw/cache_test.go b/service/cachemdw/cache_test.go index c7cdd7b..8891611 100644 --- a/service/cachemdw/cache_test.go +++ b/service/cachemdw/cache_test.go @@ -35,10 +35,6 @@ func NewMockEVMBlockGetter() *MockEVMBlockGetter { var _ decode.EVMBlockGetter = (*MockEVMBlockGetter)(nil) -func (c *MockEVMBlockGetter) BlockByHash(ctx context.Context, hash common.Hash) (*ethctypes.Block, error) { - panic("not implemented") -} - func (c *MockEVMBlockGetter) HeaderByHash(ctx context.Context, hash common.Hash) (*ethctypes.Header, error) { panic("not implemented") } From 07b4347682ead560ca4045d0f5c057890be4d533 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Thu, 19 Oct 2023 20:51:58 -0400 Subject: [PATCH 24/29] Improve IsCacheable method --- service/cachemdw/cache.go | 37 ++++++++++++++++---------- service/cachemdw/cache_test.go | 9 ++----- service/cachemdw/caching_middleware.go | 2 +- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/service/cachemdw/cache.go b/service/cachemdw/cache.go index 0f2874e..0790166 100644 --- a/service/cachemdw/cache.go +++ b/service/cachemdw/cache.go @@ -49,27 +49,36 @@ func NewServiceCache( // IsCacheable checks if EVM request is cacheable. // In current implementation we consider request is cacheable if it has specific block height func IsCacheable( - ctx context.Context, - blockGetter decode.EVMBlockGetter, logger *logging.ServiceLogger, req *decode.EVMRPCRequestEnvelope, ) bool { - blockNumber, err := req.ExtractBlockNumberFromEVMRPCRequest(ctx, blockGetter) - if err != nil { - logger.Logger.Error(). - Err(err). - Msg("can't extract block number from EVM RPC request") + if req.Method == "" { return false } - // blockNumber <= 0 means magic tag was used, one of the "latest", "pending", "earliest", etc... - // as of now we don't cache requests with magic tags - if blockNumber <= 0 { - return false + if decode.MethodRequiresNoHistory(req.Method) { + return true + } + + if decode.MethodHasBlockHashParam(req.Method) { + return true + } + + if decode.MethodHasBlockNumberParam(req.Method) { + blockNumber, err := decode.ParseBlockNumberFromParams(req.Method, req.Params) + if err != nil { + logger.Logger.Error(). + Err(err). + Msg("can't parse block number from params") + return false + } + + // blockNumber < 0 means magic tag was used, one of the "latest", "pending", "earliest", etc... + // we cache requests without magic tag or with the earliest magic tag + return blockNumber > 0 || blockNumber == decode.BlockTagToNumberCodec[decode.BlockTagEarliest] } - // block number is specified and it's not a magic tag - cache the request - return true + return false } // GetCachedQueryResponse calculates cache key for request and then tries to get it from cache. @@ -97,7 +106,7 @@ func (c *ServiceCache) CacheQueryResponse( responseInBytes []byte, ) error { // don't cache uncacheable requests - if !IsCacheable(ctx, c.blockGetter, c.ServiceLogger, req) { + if !IsCacheable(c.ServiceLogger, req) { return errors.New("query isn't cacheable") } diff --git a/service/cachemdw/cache_test.go b/service/cachemdw/cache_test.go index 8891611..6743025 100644 --- a/service/cachemdw/cache_test.go +++ b/service/cachemdw/cache_test.go @@ -2,7 +2,6 @@ package cachemdw_test import ( "context" - "math/big" "testing" "time" @@ -23,8 +22,7 @@ const ( ) var ( - defaultCachePrefix = big.NewInt(1) - defaultQueryResp = []byte(testEVMQueries[TestRequestWeb3ClientVersion].ResponseBody) + defaultQueryResp = []byte(testEVMQueries[TestRequestWeb3ClientVersion].ResponseBody) ) type MockEVMBlockGetter struct{} @@ -43,9 +41,6 @@ func TestUnitTestIsCacheable(t *testing.T) { logger, err := logging.New("TRACE") require.NoError(t, err) - blockGetter := NewMockEVMBlockGetter() - ctxb := context.Background() - for _, tc := range []struct { desc string req *decode.EVMRPCRequestEnvelope @@ -63,7 +58,7 @@ func TestUnitTestIsCacheable(t *testing.T) { }, } { t.Run(tc.desc, func(t *testing.T) { - cacheable := cachemdw.IsCacheable(ctxb, blockGetter, &logger, tc.req) + cacheable := cachemdw.IsCacheable(&logger, tc.req) require.Equal(t, tc.cacheable, cacheable) }) } diff --git a/service/cachemdw/caching_middleware.go b/service/cachemdw/caching_middleware.go index 547b3a1..2687bcc 100644 --- a/service/cachemdw/caching_middleware.go +++ b/service/cachemdw/caching_middleware.go @@ -40,7 +40,7 @@ func (c *ServiceCache) CachingMiddleware( } isCached := IsRequestCached(r.Context()) - cacheable := IsCacheable(r.Context(), c.blockGetter, c.ServiceLogger, decodedReq) + cacheable := IsCacheable(c.ServiceLogger, decodedReq) response := r.Context().Value(ResponseContextKey) typedResponse, ok := response.([]byte) From 4e5db11cc3537bcc0c5e990824784f680393007d Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 20 Oct 2023 10:35:22 -0400 Subject: [PATCH 25/29] Improve logging --- service/cachemdw/caching_middleware.go | 6 ++++++ service/cachemdw/is_cached_middleware.go | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/service/cachemdw/caching_middleware.go b/service/cachemdw/caching_middleware.go index 2687bcc..63f237c 100644 --- a/service/cachemdw/caching_middleware.go +++ b/service/cachemdw/caching_middleware.go @@ -21,6 +21,12 @@ func (c *ServiceCache) CachingMiddleware( return func(w http.ResponseWriter, r *http.Request) { // if cache is not enabled - do nothing and forward to next middleware if !c.cacheEnabled { + c.Logger.Trace(). + Str("method", r.Method). + Str("url", r.URL.String()). + Str("host", r.Host). + Msg("cache is disabled skipping caching-middleware") + next.ServeHTTP(w, r) return } diff --git a/service/cachemdw/is_cached_middleware.go b/service/cachemdw/is_cached_middleware.go index 6936556..64a3b6c 100644 --- a/service/cachemdw/is_cached_middleware.go +++ b/service/cachemdw/is_cached_middleware.go @@ -30,6 +30,12 @@ func (c *ServiceCache) IsCachedMiddleware( return func(w http.ResponseWriter, r *http.Request) { // if cache is not enabled - do nothing and forward to next middleware if !c.cacheEnabled { + c.Logger.Trace(). + Str("method", r.Method). + Str("url", r.URL.String()). + Str("host", r.Host). + Msg("cache is disabled skipping is-cached-middleware") + next.ServeHTTP(w, r) return } From f9fbfe54063d4f1a62dff7b7285b03ffd4d35478 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 20 Oct 2023 10:40:05 -0400 Subject: [PATCH 26/29] CR's fixes --- .env | 6 +++--- config/config.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index 7293f19..c7c2375 100644 --- a/.env +++ b/.env @@ -107,9 +107,9 @@ CACHE_ENABLED=true # REDIS_ENDPOINT_URL is an url of redis REDIS_ENDPOINT_URL=redis:6379 REDIS_PASSWORD= -# CACHE_TTL is a TTL for cached evm requests -# CACHE_TTL should be specified in seconds -CACHE_TTL=600 +# CACHE_TTL_SECONDS is a TTL for cached evm requests +# CACHE_TTL_SECONDS should be specified in seconds +CACHE_TTL_SECONDS=600 # CACHE_PREFIX is used as prefix for any key in the cache, key has such structure: # :evm-request::sha256: # Possible values are testnet, mainnet, etc... diff --git a/config/config.go b/config/config.go index 42dee3f..f9593a6 100644 --- a/config/config.go +++ b/config/config.go @@ -92,7 +92,7 @@ const ( CACHE_ENABLED_ENVIRONMENT_KEY = "CACHE_ENABLED" REDIS_ENDPOINT_URL_ENVIRONMENT_KEY = "REDIS_ENDPOINT_URL" REDIS_PASSWORD_ENVIRONMENT_KEY = "REDIS_PASSWORD" - CACHE_TTL_ENVIRONMENT_KEY = "CACHE_TTL" + CACHE_TTL_ENVIRONMENT_KEY = "CACHE_TTL_SECONDS" CACHE_PREFIX_ENVIRONMENT_KEY = "CACHE_PREFIX" ) From 00ed140d8c7825ebbd06fe6f03572350722e420e Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 20 Oct 2023 11:15:10 -0400 Subject: [PATCH 27/29] Fix JSON-RPC response's ID flow --- main_test.go | 93 ++++++++++++++++++++++++++++++---- service/cachemdw/cache.go | 26 ++++++++-- service/cachemdw/cache_test.go | 2 +- 3 files changed, 108 insertions(+), 13 deletions(-) diff --git a/main_test.go b/main_test.go index c73c268..a50161a 100644 --- a/main_test.go +++ b/main_test.go @@ -480,7 +480,7 @@ func TestE2ETestCachingMdwWithBlockNumberParam(t *testing.T) { // check that cached and non-cached responses are equal // eth_getBlockByNumber - cache MISS - resp1 := mkJsonRpcRequest(t, proxyServiceURL, tc.method, tc.params) + resp1 := mkJsonRpcRequest(t, proxyServiceURL, 1, tc.method, tc.params) require.Equal(t, cachemdw.CacheMissHeaderValue, resp1.Header[cachemdw.CacheHeaderKey][0]) body1, err := io.ReadAll(resp1.Body) require.NoError(t, err) @@ -491,7 +491,7 @@ func TestE2ETestCachingMdwWithBlockNumberParam(t *testing.T) { containsKey(t, redisClient, expectedKey) // eth_getBlockByNumber - cache HIT - resp2 := mkJsonRpcRequest(t, proxyServiceURL, tc.method, tc.params) + resp2 := mkJsonRpcRequest(t, proxyServiceURL, 1, tc.method, tc.params) require.Equal(t, cachemdw.CacheHitHeaderValue, resp2.Header[cachemdw.CacheHeaderKey][0]) body2, err := io.ReadAll(resp2.Body) require.NoError(t, err) @@ -563,7 +563,7 @@ func TestE2ETestCachingMdwWithBlockNumberParam_EmptyResult(t *testing.T) { // check that responses are equal // eth_getBlockByNumber - cache MISS - resp1 := mkJsonRpcRequest(t, proxyServiceURL, tc.method, tc.params) + resp1 := mkJsonRpcRequest(t, proxyServiceURL, 1, tc.method, tc.params) require.Equal(t, cachemdw.CacheMissHeaderValue, resp1.Header[cachemdw.CacheHeaderKey][0]) body1, err := io.ReadAll(resp1.Body) require.NoError(t, err) @@ -572,7 +572,7 @@ func TestE2ETestCachingMdwWithBlockNumberParam_EmptyResult(t *testing.T) { expectKeysNum(t, redisClient, tc.keysNum) // eth_getBlockByNumber - cache MISS again (empty results aren't cached) - resp2 := mkJsonRpcRequest(t, proxyServiceURL, tc.method, tc.params) + resp2 := mkJsonRpcRequest(t, proxyServiceURL, 1, tc.method, tc.params) require.Equal(t, cachemdw.CacheMissHeaderValue, resp2.Header[cachemdw.CacheHeaderKey][0]) body2, err := io.ReadAll(resp2.Body) require.NoError(t, err) @@ -603,6 +603,81 @@ func TestE2ETestCachingMdwWithBlockNumberParam_EmptyResult(t *testing.T) { cleanUpRedis(t, redisClient) } +func TestE2ETestCachingMdwWithBlockNumberParam_DiffJsonRpcReqIDs(t *testing.T) { + redisClient := redis.NewClient(&redis.Options{ + Addr: redisURL, + Password: redisPassword, + DB: 0, + }) + cleanUpRedis(t, redisClient) + expectKeysNum(t, redisClient, 0) + + for _, tc := range []struct { + desc string + method string + params []interface{} + keysNum int + }{ + { + desc: "test case #1", + method: "eth_getBlockByNumber", + params: []interface{}{"0x1", true}, + keysNum: 1, + }, + } { + t.Run(tc.desc, func(t *testing.T) { + // test cache MISS and cache HIT scenarios for specified method + // check corresponding values in cachemdw.CacheHeaderKey HTTP header + // NOTE: JSON-RPC request IDs are different + // check that cached and non-cached responses differ only in response ID + + // eth_getBlockByNumber - cache MISS + resp1 := mkJsonRpcRequest(t, proxyServiceURL, 1, tc.method, tc.params) + require.Equal(t, cachemdw.CacheMissHeaderValue, resp1.Header[cachemdw.CacheHeaderKey][0]) + body1, err := io.ReadAll(resp1.Body) + require.NoError(t, err) + err = checkJsonRpcErr(body1) + require.NoError(t, err) + expectKeysNum(t, redisClient, tc.keysNum) + expectedKey := "local-chain:evm-request:eth_getBlockByNumber:sha256:d08b426164eacf6646fb1817403ec0af5d37869a0f32a01ebfab3096fa4999be" + containsKey(t, redisClient, expectedKey) + + // eth_getBlockByNumber - cache HIT + resp2 := mkJsonRpcRequest(t, proxyServiceURL, 2, tc.method, tc.params) + require.Equal(t, cachemdw.CacheHitHeaderValue, resp2.Header[cachemdw.CacheHeaderKey][0]) + body2, err := io.ReadAll(resp2.Body) + require.NoError(t, err) + err = checkJsonRpcErr(body2) + require.NoError(t, err) + expectKeysNum(t, redisClient, tc.keysNum) + containsKey(t, redisClient, expectedKey) + + rpcResp1, err := cachemdw.UnmarshalJsonRpcResponse(body1) + require.NoError(t, err) + rpcResp2, err := cachemdw.UnmarshalJsonRpcResponse(body2) + require.NoError(t, err) + + // JSON-RPC Version and Result should be equal + require.Equal(t, rpcResp1.Version, rpcResp2.Version) + require.Equal(t, rpcResp1.Result, rpcResp2.Result) + + // JSON-RPC response ID should correspond to JSON-RPC request ID + require.Equal(t, string(rpcResp1.ID), "1") + require.Equal(t, string(rpcResp2.ID), "2") + + // JSON-RPC error should be empty + require.Empty(t, rpcResp1.JsonRpcError) + require.Empty(t, rpcResp2.JsonRpcError) + + // Double-check that JSON-RPC responses differ only in response ID + rpcResp2.ID = []byte("1") + require.Equal(t, rpcResp1, rpcResp2) + }) + } + + cleanUpRedis(t, redisClient) +} + func expectKeysNum(t *testing.T, redisClient *redis.Client, keysNum int) { keys, err := redisClient.Keys(context.Background(), "*").Result() require.NoError(t, err) @@ -626,8 +701,8 @@ func cleanUpRedis(t *testing.T, redisClient *redis.Client) { } } -func mkJsonRpcRequest(t *testing.T, proxyServiceURL, method string, params []interface{}) *http.Response { - req := newJsonRpcRequest(method, params) +func mkJsonRpcRequest(t *testing.T, proxyServiceURL string, id int, method string, params []interface{}) *http.Response { + req := newJsonRpcRequest(id, method, params) reqInJSON, err := json.Marshal(req) require.NoError(t, err) reqReader := bytes.NewBuffer(reqInJSON) @@ -640,17 +715,17 @@ func mkJsonRpcRequest(t *testing.T, proxyServiceURL, method string, params []int type jsonRpcRequest struct { JsonRpc string `json:"jsonrpc"` + Id int `json:"id"` Method string `json:"method"` Params []interface{} `json:"params"` - Id int `json:"id"` } -func newJsonRpcRequest(method string, params []interface{}) *jsonRpcRequest { +func newJsonRpcRequest(id int, method string, params []interface{}) *jsonRpcRequest { return &jsonRpcRequest{ JsonRpc: "2.0", + Id: id, Method: method, Params: params, - Id: 1, } } diff --git a/service/cachemdw/cache.go b/service/cachemdw/cache.go index 0790166..7115554 100644 --- a/service/cachemdw/cache.go +++ b/service/cachemdw/cache.go @@ -2,8 +2,10 @@ package cachemdw import ( "context" + "encoding/json" "errors" "fmt" + "strconv" "time" "github.com/kava-labs/kava-proxy-service/clients/cache" @@ -82,6 +84,8 @@ func IsCacheable( } // GetCachedQueryResponse calculates cache key for request and then tries to get it from cache. +// NOTE: only JSON-RPC response's result will be taken from the cache. +// JSON-RPC response's ID and Version will be constructed on the fly to match JSON-RPC request. func (c *ServiceCache) GetCachedQueryResponse( ctx context.Context, req *decode.EVMRPCRequestEnvelope, @@ -91,15 +95,31 @@ func (c *ServiceCache) GetCachedQueryResponse( return nil, err } - value, err := c.cacheClient.Get(ctx, key) + // get JSON-RPC response's result from the cache + result, err := c.cacheClient.Get(ctx, key) if err != nil { return nil, err } - return value, nil + // JSON-RPC response's ID and Version should match JSON-RPC request + id := strconv.Itoa(int(req.ID)) + response := JsonRpcResponse{ + Version: req.JSONRPCVersion, + ID: []byte(id), + Result: result, + } + responseInJSON, err := json.Marshal(response) + if err != nil { + return nil, err + } + + return responseInJSON, nil } // CacheQueryResponse calculates cache key for request and then saves response to the cache. +// NOTE: only JSON-RPC response's result is cached. +// There is no point to cache JSON-RPC response's ID (because it should correspond to request's ID, which constantly changes). +// Same with JSON-RPC response's Version. func (c *ServiceCache) CacheQueryResponse( ctx context.Context, req *decode.EVMRPCRequestEnvelope, @@ -124,7 +144,7 @@ func (c *ServiceCache) CacheQueryResponse( return err } - return c.cacheClient.Set(ctx, key, responseInBytes, c.cacheTTL) + return c.cacheClient.Set(ctx, key, response.Result, c.cacheTTL) } func (c *ServiceCache) Healthcheck(ctx context.Context) error { diff --git a/service/cachemdw/cache_test.go b/service/cachemdw/cache_test.go index 6743025..3d39e5c 100644 --- a/service/cachemdw/cache_test.go +++ b/service/cachemdw/cache_test.go @@ -93,7 +93,7 @@ func TestUnitTestCacheQueryResponse(t *testing.T) { resp, err = serviceCache.GetCachedQueryResponse(ctxb, req) require.NoError(t, err) - require.Equal(t, defaultQueryResp, resp) + require.JSONEq(t, string(defaultQueryResp), string(resp)) } func mkEVMRPCRequestEnvelope(blockNumber string) *decode.EVMRPCRequestEnvelope { From 6cb7f29bd7070c38699d05f94c71ac622a1c7c26 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 20 Oct 2023 13:38:54 -0400 Subject: [PATCH 28/29] Remove deprecated TODO --- service/cachemdw/keys.go | 1 - 1 file changed, 1 deletion(-) diff --git a/service/cachemdw/keys.go b/service/cachemdw/keys.go index fb83a0c..7cd5ead 100644 --- a/service/cachemdw/keys.go +++ b/service/cachemdw/keys.go @@ -46,7 +46,6 @@ func GetQueryKey( return "", fmt.Errorf("request shouldn't be nil") } - // TODO(yevhenii): use stable/sorted JSON serializer serializedParams, err := json.Marshal(req.Params) if err != nil { return "", err From 064faf803f8062ebe68470720687a91072b0ac37 Mon Sep 17 00:00:00 2001 From: evgeniy-scherbina Date: Fri, 20 Oct 2023 13:47:19 -0400 Subject: [PATCH 29/29] Small fix --- service/cachemdw/cache.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/service/cachemdw/cache.go b/service/cachemdw/cache.go index 7115554..17f597f 100644 --- a/service/cachemdw/cache.go +++ b/service/cachemdw/cache.go @@ -57,11 +57,7 @@ func IsCacheable( if req.Method == "" { return false } - - if decode.MethodRequiresNoHistory(req.Method) { - return true - } - + if decode.MethodHasBlockHashParam(req.Method) { return true }