From bc65a1fab4f0b35356278a550d703f7bc1497625 Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Mon, 23 Dec 2024 16:05:16 +0200 Subject: [PATCH 01/11] rework cache --- go.mod | 9 +- go.sum | 26 +-- go/common/gethencoding/geth_encoding.go | 2 +- go/enclave/rpc/GetTransactionReceipt.go | 6 + go/enclave/storage/cache_service.go | 210 ++++++++---------- go/enclave/storage/events_storage.go | 10 +- go/enclave/storage/storage.go | 26 +-- tools/walletextension/cache/RistrettoCache.go | 8 +- tools/walletextension/cache/cache.go | 2 +- 9 files changed, 134 insertions(+), 165 deletions(-) diff --git a/go.mod b/go.mod index 1b342c162..2cabde496 100644 --- a/go.mod +++ b/go.mod @@ -12,12 +12,10 @@ require ( github.com/andybalholm/brotli v1.1.1 github.com/codeclysm/extract/v3 v3.1.1 github.com/deckarep/golang-set/v2 v2.6.0 - github.com/dgraph-io/ristretto v0.2.0 + github.com/dgraph-io/ristretto/v2 v2.0.1 github.com/docker/docker v25.0.4+incompatible github.com/docker/go-connections v0.5.0 github.com/edgelesssys/ego v1.6.0 - github.com/eko/gocache/lib/v4 v4.1.6 - github.com/eko/gocache/store/ristretto/v4 v4.2.1 github.com/ethereum/go-ethereum v1.14.6 github.com/gin-contrib/cors v1.7.2 github.com/gin-gonic/gin v1.10.0 @@ -43,7 +41,7 @@ require ( github.com/sanity-io/litter v1.5.5 github.com/spf13/viper v1.19.0 github.com/status-im/keycard-go v0.3.2 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/tidwall/gjson v1.18.0 github.com/urfave/cli/v2 v2.27.5 github.com/valyala/fasthttp v1.57.0 @@ -108,7 +106,6 @@ require ( github.com/go-playground/validator/v10 v10.22.1 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/mock v1.6.0 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/h2non/filetype v1.1.3 // indirect @@ -172,7 +169,7 @@ require ( go.uber.org/multierr v1.9.0 // indirect golang.org/x/arch v0.11.0 // indirect golang.org/x/net v0.30.0 // indirect - golang.org/x/sys v0.27.0 // indirect + golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.20.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 3267e8a8a..b5cbd7104 100644 --- a/go.sum +++ b/go.sum @@ -91,8 +91,8 @@ github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5il github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dru4WE= -github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU= +github.com/dgraph-io/ristretto/v2 v2.0.1 h1:7W0LfEP+USCmtrUjJsk+Jv2jbhJmb72N4yRI7GrLdMI= +github.com/dgraph-io/ristretto/v2 v2.0.1/go.mod h1:K7caLeufSdxm+ITp1n/73U+VbFVAHrexfLbz4n14hpo= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= @@ -109,10 +109,6 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/edgelesssys/ego v1.6.0 h1:QfArp8vUwsH1G4966ZQv5i3f1BxsbIP2uOKkIGjsGyQ= github.com/edgelesssys/ego v1.6.0/go.mod h1:XAej1u86vecy/UYGxMxDP37LTHTPC3/7cbqNodwcTXE= -github.com/eko/gocache/lib/v4 v4.1.6 h1:5WWIGISKhE7mfkyF+SJyWwqa4Dp2mkdX8QsZpnENqJI= -github.com/eko/gocache/lib/v4 v4.1.6/go.mod h1:HFxC8IiG2WeRotg09xEnPD72sCheJiTSr4Li5Ameg7g= -github.com/eko/gocache/store/ristretto/v4 v4.2.1 h1:xB5E1LP1gh8yUV1G3KVRSL4T0OTnxp4OixuTljn2848= -github.com/eko/gocache/store/ristretto/v4 v4.2.1/go.mod h1:KyshDyWQqfSVrg2rH06fFQZTj6vG2fxlY7oAW9oxNHY= github.com/ethereum/c-kzg-4844 v1.0.3 h1:IEnbOHwjixW2cTvKRUlAAUOeleV7nNM/umJR+qy4WDs= github.com/ethereum/c-kzg-4844 v1.0.3/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.14.6 h1:ZTxnErSopkDyxdvB8zW/KcK+/AVrdil/TzoWXVKaaC8= @@ -176,8 +172,6 @@ github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOW github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -360,8 +354,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/supranational/blst v0.3.13 h1:AYeSxdOMacwu7FBmpfloBz5pbFXDmJL33RuwnKtmTjk= @@ -399,7 +393,6 @@ github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZ github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40 h1:dizWJqTWjwyD8KGcMOwgrkqu1JIkofYgKkmDeNE7oAs= @@ -419,7 +412,6 @@ golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 h1:6R2FC06FonbXQ8pK11/PDFY6N golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -427,7 +419,6 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -448,9 +439,6 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -458,9 +446,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -472,7 +459,6 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/go/common/gethencoding/geth_encoding.go b/go/common/gethencoding/geth_encoding.go index 50b04a669..57c5658b6 100644 --- a/go/common/gethencoding/geth_encoding.go +++ b/go/common/gethencoding/geth_encoding.go @@ -265,7 +265,7 @@ func ExtractEthCall(param interface{}) (*gethapi.TransactionArgs, error) { // Special care must be taken to maintain a valid chain of these converted headers. func (enc *gethEncodingServiceImpl) CreateEthHeaderForBatch(ctx context.Context, h *common.BatchHeader) (*types.Header, error) { // wrap in a caching layer - return enc.cachingService.ReadConvertedHeader(ctx, h.Hash(), func(a any) (*types.Header, error) { + return enc.cachingService.ReadConvertedHeader(ctx, h.Hash(), func() (*types.Header, error) { // deterministically calculate the private randomness that will be exposed to the EVM perBatchRandomness := enc.entropyService.BatchEntropy(h.Number) diff --git a/go/enclave/rpc/GetTransactionReceipt.go b/go/enclave/rpc/GetTransactionReceipt.go index 783a086c3..f051a5d76 100644 --- a/go/enclave/rpc/GetTransactionReceipt.go +++ b/go/enclave/rpc/GetTransactionReceipt.go @@ -114,6 +114,12 @@ func fetchFromCache(ctx context.Context, storage storage.Storage, cacheService * } } r := marshalReceipt(rec.Receipt, logs, rec.From, rec.To) + + // after the receipt was requested by a user remove it from the cache + err := cacheService.DelReceipt(ctx, txHash) + if err != nil { + return nil, err + } return r, nil } diff --git a/go/enclave/storage/cache_service.go b/go/enclave/storage/cache_service.go index 59fbf5369..e4775df24 100644 --- a/go/enclave/storage/cache_service.go +++ b/go/enclave/storage/cache_service.go @@ -3,21 +3,16 @@ package storage import ( "context" "encoding/binary" - "fmt" "math/big" + "time" - "github.com/status-im/keycard-go/hexutils" + "github.com/ten-protocol/go-ten/go/common/errutil" "github.com/ten-protocol/go-ten/go/enclave/storage/enclavedb" - "github.com/eko/gocache/lib/v4/store" - "github.com/ten-protocol/go-ten/go/enclave/core" - "github.com/dgraph-io/ristretto" - "github.com/eko/gocache/lib/v4/cache" - ristretto_store "github.com/eko/gocache/store/ristretto/v4" - + "github.com/dgraph-io/ristretto/v2" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" gethlog "github.com/ethereum/go-ethereum/log" @@ -41,46 +36,46 @@ const ( type CacheService struct { // cache for the immutable blocks headers - blockCache *cache.Cache[*types.Header] + blockCache *ristretto.Cache[[]byte, *types.Header] // stores batches using the sequence number as key - batchCacheBySeqNo *cache.Cache[*common.BatchHeader] + batchCacheBySeqNo *ristretto.Cache[uint64, *common.BatchHeader] // mapping between the hash and the sequence number // note: to fetch a batch by hash will require 2 cache hits - seqCacheByHash *cache.Cache[*big.Int] + seqCacheByHash *ristretto.Cache[[]byte, *big.Int] // mapping between the height and the sequence number // note: to fetch a batch by height will require 2 cache hits - seqCacheByHeight *cache.Cache[*big.Int] + seqCacheByHeight *ristretto.Cache[uint64, *big.Int] // store the converted ethereum header which is passed to the evm - convertedGethHeaderCache *cache.Cache[*types.Header] + convertedGethHeaderCache *ristretto.Cache[[]byte, *types.Header] // batch hash - geth converted hash - convertedHashCache *cache.Cache[*gethcommon.Hash] + convertedHashCache *ristretto.Cache[[]byte, *gethcommon.Hash] // from address ( either eoa or contract) to the id of the db entry - eoaCache *cache.Cache[*uint64] - contractAddressCache *cache.Cache[*enclavedb.Contract] - eventTopicCache *cache.Cache[*enclavedb.EventTopic] + eoaCache *ristretto.Cache[[]byte, *uint64] + contractAddressCache *ristretto.Cache[[]byte, *enclavedb.Contract] + eventTopicCache *ristretto.Cache[[]byte, *enclavedb.EventTopic] // from contract_address||event_sig to the event_type object - eventTypeCache *cache.Cache[*enclavedb.EventType] + eventTypeCache *ristretto.Cache[[]byte, *enclavedb.EventType] // store the last few batches together with the content - lastBatchesCache *cache.Cache[*core.Batch] + lastBatchesCache *ristretto.Cache[uint64, *core.Batch] // store all recent receipts in a cache // together with the sender - and for each log whether it is visible by the sender // only sender can view configured - receiptCache *cache.Cache[*CachedReceipt] + receiptCache *ristretto.Cache[[]byte, *CachedReceipt] // store the enclaves from the network - attestedEnclavesCache *cache.Cache[*AttestedEnclave] + attestedEnclavesCache *ristretto.Cache[[]byte, *AttestedEnclave] - // cache for sequencer enclave IDs (using a dummy key since it's a global list) - sequencerIDsCache *cache.Cache[*[]common.EnclaveID] + // cache for sequencer enclave IDs + sequencerIDsCache []common.EnclaveID logger gethlog.Logger } @@ -104,34 +99,34 @@ func NewCacheService(logger gethlog.Logger, testMode bool) *CacheService { } return &CacheService{ - blockCache: cache.New[*types.Header](newCache(logger, nrL1Blocks, blockHeaderCost)), + blockCache: newCache[[]byte, *types.Header](logger, nrL1Blocks, blockHeaderCost), - batchCacheBySeqNo: cache.New[*common.BatchHeader](newCache(logger, nrBatches, batchHeaderCost)), - seqCacheByHash: cache.New[*big.Int](newCache(logger, nrBatches, idCost)), - seqCacheByHeight: cache.New[*big.Int](newCache(logger, nrBatches, idCost)), + batchCacheBySeqNo: newCache[uint64, *common.BatchHeader](logger, nrBatches, batchHeaderCost), + seqCacheByHash: newCache[[]byte, *big.Int](logger, nrBatches, idCost), + seqCacheByHeight: newCache[uint64, *big.Int](logger, nrBatches, idCost), - convertedGethHeaderCache: cache.New[*types.Header](newCache(logger, nrConvertedEth, batchHeaderCost)), - convertedHashCache: cache.New[*gethcommon.Hash](newCache(logger, nrConvertedEth, hashCost)), + convertedGethHeaderCache: newCache[[]byte, *types.Header](logger, nrConvertedEth, batchHeaderCost), + convertedHashCache: newCache[[]byte, *gethcommon.Hash](logger, nrConvertedEth, hashCost), - eoaCache: cache.New[*uint64](newCache(logger, nrEOA, idCost)), - contractAddressCache: cache.New[*enclavedb.Contract](newCache(logger, nrContractAddresses, contractCost)), - eventTypeCache: cache.New[*enclavedb.EventType](newCache(logger, nrEventTypes, eventTypeCost)), - eventTopicCache: cache.New[*enclavedb.EventTopic](newCache(logger, nrEventTypes, eventTopicCost)), + eoaCache: newCache[[]byte, *uint64](logger, nrEOA, idCost), + contractAddressCache: newCache[[]byte, *enclavedb.Contract](logger, nrContractAddresses, contractCost), + eventTypeCache: newCache[[]byte, *enclavedb.EventType](logger, nrEventTypes, eventTypeCost), + eventTopicCache: newCache[[]byte, *enclavedb.EventTopic](logger, nrEventTypes, eventTopicCost), - receiptCache: cache.New[*CachedReceipt](newCache(logger, nrReceipts, receiptCost)), - attestedEnclavesCache: cache.New[*AttestedEnclave](newCache(logger, nrEnclaves, enclaveCost)), + receiptCache: newCache[[]byte, *CachedReceipt](logger, nrReceipts, receiptCost), + attestedEnclavesCache: newCache[[]byte, *AttestedEnclave](logger, nrEnclaves, enclaveCost), // cache the latest received batches to avoid a lookup when streaming it back to the host after processing - lastBatchesCache: cache.New[*core.Batch](newCache(logger, nrBatchesWithContent, batchCost)), + lastBatchesCache: newCache[uint64, *core.Batch](logger, nrBatchesWithContent, batchCost), - sequencerIDsCache: cache.New[*[]common.EnclaveID](newCache(logger, 1, enclaveCost*20)), // space for ~20 sequencers + sequencerIDsCache: make([]common.EnclaveID, 0), logger: logger, } } -func newCache(logger gethlog.Logger, nrElem, capacityPerElem int) *ristretto_store.RistrettoStore { - ristrettoCache, err := ristretto.NewCache(&ristretto.Config{ +func newCache[K ristretto.Key, V any](logger gethlog.Logger, nrElem, capacityPerElem int) *ristretto.Cache[K, V] { + ristrettoCache, err := ristretto.NewCache(&ristretto.Config[K, V]{ NumCounters: int64(10 * nrElem), // 10 times the expected elements MaxCost: int64(capacityPerElem * nrElem * 2), // calculate the max cost BufferItems: 64, // number of keys per Get buffer. @@ -140,65 +135,66 @@ func newCache(logger gethlog.Logger, nrElem, capacityPerElem int) *ristretto_sto if err != nil { logger.Crit("Could not initialise ristretto cache", log.ErrKey, err) } - return ristretto_store.NewRistretto(ristrettoCache) + return ristrettoCache } func (cs *CacheService) CacheConvertedHash(ctx context.Context, batchHash, convertedHash gethcommon.Hash) { - cacheValue(ctx, cs.convertedHashCache, cs.logger, batchHash, &convertedHash, hashCost) + cacheValue(ctx, cs.convertedHashCache, cs.logger, batchHash.Bytes(), &convertedHash, hashCost) } func (cs *CacheService) CacheBlock(ctx context.Context, b *types.Header) { - cacheValue(ctx, cs.blockCache, cs.logger, b.Hash(), b, blockHeaderCost) + cacheValue(ctx, cs.blockCache, cs.logger, b.Hash().Bytes(), b, blockHeaderCost) } -func (cs *CacheService) CacheReceipt(ctx context.Context, r *CachedReceipt) { - cacheValue(ctx, cs.receiptCache, cs.logger, r.Receipt.TxHash, r, receiptCost) +func (cs *CacheService) CacheReceipt(_ context.Context, r *CachedReceipt) { + // keep the receipts in cache for 1 minute + cs.receiptCache.SetWithTTL(r.Receipt.TxHash.Bytes(), r, receiptCost, 1*time.Minute) } func (cs *CacheService) CacheBatch(ctx context.Context, batch *core.Batch) { cacheValue(ctx, cs.batchCacheBySeqNo, cs.logger, batch.SeqNo().Uint64(), batch.Header, batchHeaderCost) - cacheValue(ctx, cs.seqCacheByHash, cs.logger, batch.Hash(), batch.SeqNo(), idCost) + cacheValue(ctx, cs.seqCacheByHash, cs.logger, batch.Hash().Bytes(), batch.SeqNo(), idCost) // note: the key is (height+1), because for some reason it doesn't like a key of 0 // should always contain the canonical batch because the cache is overwritten by each new batch after a reorg cacheValue(ctx, cs.seqCacheByHeight, cs.logger, batch.NumberU64()+1, batch.SeqNo(), idCost) - cacheValue(ctx, cs.lastBatchesCache, cs.logger, batch.SeqNo(), batch, batchCost) + cacheValue(ctx, cs.lastBatchesCache, cs.logger, batch.SeqNo().Uint64(), batch, batchCost) } -func (cs *CacheService) ReadBlock(ctx context.Context, key gethcommon.Hash, onCacheMiss func(any) (*types.Header, error)) (*types.Header, error) { - return getCachedValue(ctx, cs.blockCache, cs.logger, key, blockHeaderCost, onCacheMiss, true) +func (cs *CacheService) ReadBlock(ctx context.Context, key gethcommon.Hash, onCacheMiss func() (*types.Header, error)) (*types.Header, error) { + return getCachedValue(ctx, cs.blockCache, cs.logger, key.Bytes(), blockHeaderCost, onCacheMiss, true) } -func (cs *CacheService) ReadBatchSeqByHash(ctx context.Context, hash common.L2BatchHash, onCacheMiss func(any) (*big.Int, error)) (*big.Int, error) { - return getCachedValue(ctx, cs.seqCacheByHash, cs.logger, hash, idCost, onCacheMiss, true) +func (cs *CacheService) ReadBatchSeqByHash(ctx context.Context, hash common.L2BatchHash, onCacheMiss func() (*big.Int, error)) (*big.Int, error) { + return getCachedValue(ctx, cs.seqCacheByHash, cs.logger, hash.Bytes(), idCost, onCacheMiss, true) } -func (cs *CacheService) ReadBatchSeqByHeight(ctx context.Context, height uint64, onCacheMiss func(any) (*big.Int, error)) (*big.Int, error) { +func (cs *CacheService) ReadBatchSeqByHeight(ctx context.Context, height uint64, onCacheMiss func() (*big.Int, error)) (*big.Int, error) { // the key is (height+1), because for some reason it doesn't like a key of 0 return getCachedValue(ctx, cs.seqCacheByHeight, cs.logger, height+1, idCost, onCacheMiss, true) } -func (cs *CacheService) ReadConvertedHash(ctx context.Context, hash common.L2BatchHash, onCacheMiss func(any) (*gethcommon.Hash, error)) (*gethcommon.Hash, error) { - return getCachedValue(ctx, cs.convertedHashCache, cs.logger, hash, hashCost, onCacheMiss, true) +func (cs *CacheService) ReadConvertedHash(ctx context.Context, hash common.L2BatchHash, onCacheMiss func() (*gethcommon.Hash, error)) (*gethcommon.Hash, error) { + return getCachedValue(ctx, cs.convertedHashCache, cs.logger, hash.Bytes(), hashCost, onCacheMiss, true) } -func (cs *CacheService) ReadBatchHeader(ctx context.Context, seqNum uint64, onCacheMiss func(any) (*common.BatchHeader, error)) (*common.BatchHeader, error) { +func (cs *CacheService) ReadBatchHeader(ctx context.Context, seqNum uint64, onCacheMiss func() (*common.BatchHeader, error)) (*common.BatchHeader, error) { return getCachedValue(ctx, cs.batchCacheBySeqNo, cs.logger, seqNum, batchHeaderCost, onCacheMiss, true) } -func (cs *CacheService) ReadBatch(ctx context.Context, seqNum uint64, onCacheMiss func(any) (*core.Batch, error)) (*core.Batch, error) { +func (cs *CacheService) ReadBatch(ctx context.Context, seqNum uint64, onCacheMiss func() (*core.Batch, error)) (*core.Batch, error) { return getCachedValue(ctx, cs.lastBatchesCache, cs.logger, seqNum, batchCost, onCacheMiss, true) } -func (cs *CacheService) ReadEOA(ctx context.Context, addr gethcommon.Address, onCacheMiss func(any) (*uint64, error)) (*uint64, error) { - return getCachedValue(ctx, cs.eoaCache, cs.logger, addr, idCost, onCacheMiss, true) +func (cs *CacheService) ReadEOA(ctx context.Context, addr gethcommon.Address, onCacheMiss func() (*uint64, error)) (*uint64, error) { + return getCachedValue(ctx, cs.eoaCache, cs.logger, addr.Bytes(), idCost, onCacheMiss, true) } -func (cs *CacheService) ReadContractAddr(ctx context.Context, addr gethcommon.Address, onCacheMiss func(any) (*enclavedb.Contract, error)) (*enclavedb.Contract, error) { - return getCachedValue(ctx, cs.contractAddressCache, cs.logger, addr, contractCost, onCacheMiss, true) +func (cs *CacheService) ReadContractAddr(ctx context.Context, addr gethcommon.Address, onCacheMiss func() (*enclavedb.Contract, error)) (*enclavedb.Contract, error) { + return getCachedValue(ctx, cs.contractAddressCache, cs.logger, addr.Bytes(), contractCost, onCacheMiss, true) } -func (cs *CacheService) ReadEventTopic(ctx context.Context, topic []byte, eventTypeId uint64, onCacheMiss func(any) (*enclavedb.EventTopic, error)) (*enclavedb.EventTopic, error) { +func (cs *CacheService) ReadEventTopic(ctx context.Context, topic []byte, eventTypeId uint64, onCacheMiss func() (*enclavedb.EventTopic, error)) (*enclavedb.EventTopic, error) { b := make([]byte, 8) binary.LittleEndian.PutUint64(b, eventTypeId) key := append(topic, b...) @@ -212,22 +208,27 @@ type CachedReceipt struct { } func (cs *CacheService) ReadReceipt(ctx context.Context, txHash gethcommon.Hash) (*CachedReceipt, error) { - return getCachedValue(ctx, cs.receiptCache, cs.logger, txHash, receiptCost, nil, false) + return getCachedValue(ctx, cs.receiptCache, cs.logger, txHash.Bytes(), receiptCost, nil, false) } -func (cs *CacheService) ReadEventType(ctx context.Context, contractAddress gethcommon.Address, eventSignature gethcommon.Hash, onCacheMiss func(any) (*enclavedb.EventType, error)) (*enclavedb.EventType, error) { +func (cs *CacheService) DelReceipt(_ context.Context, txHash gethcommon.Hash) error { + cs.receiptCache.Del(txHash.Bytes()) + return nil +} + +func (cs *CacheService) ReadEventType(ctx context.Context, contractAddress gethcommon.Address, eventSignature gethcommon.Hash, onCacheMiss func() (*enclavedb.EventType, error)) (*enclavedb.EventType, error) { key := make([]byte, 0) key = append(key, contractAddress.Bytes()...) key = append(key, eventSignature.Bytes()...) return getCachedValue(ctx, cs.eventTypeCache, cs.logger, key, eventTypeCost, onCacheMiss, true) } -func (cs *CacheService) ReadConvertedHeader(ctx context.Context, batchHash common.L2BatchHash, onCacheMiss func(any) (*types.Header, error)) (*types.Header, error) { - return getCachedValue(ctx, cs.convertedGethHeaderCache, cs.logger, batchHash, blockHeaderCost, onCacheMiss, true) +func (cs *CacheService) ReadConvertedHeader(ctx context.Context, batchHash common.L2BatchHash, onCacheMiss func() (*types.Header, error)) (*types.Header, error) { + return getCachedValue(ctx, cs.convertedGethHeaderCache, cs.logger, batchHash.Bytes(), blockHeaderCost, onCacheMiss, true) } -func (cs *CacheService) ReadEnclavePubKey(ctx context.Context, enclaveId common.EnclaveID, onCacheMiss func(any) (*AttestedEnclave, error)) (*AttestedEnclave, error) { - return getCachedValue(ctx, cs.attestedEnclavesCache, cs.logger, enclaveId, enclaveCost, onCacheMiss, true) +func (cs *CacheService) ReadEnclavePubKey(ctx context.Context, enclaveId common.EnclaveID, onCacheMiss func() (*AttestedEnclave, error)) (*AttestedEnclave, error) { + return getCachedValue(ctx, cs.attestedEnclavesCache, cs.logger, enclaveId.Bytes(), enclaveCost, onCacheMiss, true) } func (cs *CacheService) UpdateEnclaveNodeType(ctx context.Context, enclaveId common.EnclaveID, nodeType common.NodeType) { @@ -237,70 +238,49 @@ func (cs *CacheService) UpdateEnclaveNodeType(ctx context.Context, enclaveId com return } enclave.Type = nodeType - cacheValue(ctx, cs.attestedEnclavesCache, cs.logger, enclaveId, enclave, enclaveCost) + cacheValue(ctx, cs.attestedEnclavesCache, cs.logger, enclaveId.Bytes(), enclave, enclaveCost) } -func (cs *CacheService) CacheSequencerIDs(ctx context.Context, sequencerIDs []common.EnclaveID) { - cacheValue(ctx, cs.sequencerIDsCache, cs.logger, "sequencers", &sequencerIDs, enclaveCost*20) +func (cs *CacheService) CacheSequencerIDs(_ context.Context, sequencerIDs []common.EnclaveID) { + cs.sequencerIDsCache = sequencerIDs } -func (cs *CacheService) ReadSequencerIDs(ctx context.Context, onCacheMiss func(any) (*[]common.EnclaveID, error)) (*[]common.EnclaveID, error) { - return getCachedValue(ctx, cs.sequencerIDsCache, cs.logger, "sequencers", enclaveCost*20, onCacheMiss, true) +func (cs *CacheService) ReadSequencerIDs(_ context.Context, onCacheMiss func() ([]common.EnclaveID, error)) ([]common.EnclaveID, error) { + if len(cs.sequencerIDsCache) == 0 { + var err error + cs.sequencerIDsCache, err = onCacheMiss() + if err != nil { + return nil, err + } + } + return cs.sequencerIDsCache, nil } // getCachedValue - returns the cached value for the provided key. If the key is not found, then invoke the 'onCacheMiss' function // which returns the value, and cache it -func getCachedValue[V any](ctx context.Context, cache *cache.Cache[*V], logger gethlog.Logger, key any, cost int64, onCacheMiss func(any) (*V, error), cacheIfMissing bool) (*V, error) { - value, err := cache.Get(ctx, toString(key)) - if onCacheMiss == nil { - return value, err +func getCachedValue[K ristretto.Key, V any](ctx context.Context, cache *ristretto.Cache[K, V], logger gethlog.Logger, key K, cost int64, onCacheMiss func() (V, error), cacheIfMissing bool) (V, error) { + value, found := cache.Get(key) + if found { + return value, nil } - if err != nil || value == nil { - // todo metrics for cache misses - v, err := onCacheMiss(key) - if err != nil { - return v, err - } - if v == nil { - logger.Crit("Returned a nil value from the onCacheMiss function. Should not happen.") - } - if cacheIfMissing { - cacheValue(ctx, cache, logger, key, v, cost) - } - return v, nil + if onCacheMiss == nil { + return value, errutil.ErrNotFound } - return value, err -} - -func cacheValue[V any](ctx context.Context, cache *cache.Cache[*V], logger gethlog.Logger, key any, v *V, cost int64) { - if v == nil { - return - } - err := cache.Set(ctx, toString(key), v, store.WithCost(cost)) + v, err := onCacheMiss() if err != nil { - logger.Error("Could not store value in cache", log.ErrKey, err) + return v, err + } + if cacheIfMissing { + cacheValue(ctx, cache, logger, key, v, cost) } + return v, nil } -// ristretto cache works with string keys -// if anything else is presented, it will use MD5 -func toString(key any) string { - switch k := key.(type) { - case string: - return k - case []byte: - return hexutils.BytesToHex(k) - case gethcommon.Hash: - return hexutils.BytesToHex(k.Bytes()) - case gethcommon.Address: - return hexutils.BytesToHex(k.Bytes()) - case uint64, int64, int, uint: - return fmt.Sprint(k) - case *big.Int: - return fmt.Sprint(k) - default: - panic("should not happen. Invalid cache type") +func cacheValue[K ristretto.Key, V any](_ context.Context, cache *ristretto.Cache[K, V], logger gethlog.Logger, key K, v V, cost int64) { + added := cache.Set(key, v, cost) + if !added { + logger.Debug("Did not store value in cache") } } diff --git a/go/enclave/storage/events_storage.go b/go/enclave/storage/events_storage.go index 484bd296a..0d4df41ff 100644 --- a/go/enclave/storage/events_storage.go +++ b/go/enclave/storage/events_storage.go @@ -308,7 +308,7 @@ func (es *eventsStorage) determineRelevantAddressForTopic(ctx context.Context, d func (es *eventsStorage) readEventType(ctx context.Context, dbTX *sql.Tx, contractAddress gethcommon.Address, eventSignature gethcommon.Hash) (*enclavedb.EventType, error) { defer es.logDuration("ReadEventType", measure.NewStopwatch()) - return es.cachingService.ReadEventType(ctx, contractAddress, eventSignature, func(v any) (*enclavedb.EventType, error) { + return es.cachingService.ReadEventType(ctx, contractAddress, eventSignature, func() (*enclavedb.EventType, error) { contract, err := es.readContract(ctx, dbTX, contractAddress) if err != nil { return nil, err @@ -319,14 +319,14 @@ func (es *eventsStorage) readEventType(ctx context.Context, dbTX *sql.Tx, contra func (es *eventsStorage) readContract(ctx context.Context, dbTX *sql.Tx, addr gethcommon.Address) (*enclavedb.Contract, error) { defer es.logDuration("readContract", measure.NewStopwatch()) - return es.cachingService.ReadContractAddr(ctx, addr, func(v any) (*enclavedb.Contract, error) { + return es.cachingService.ReadContractAddr(ctx, addr, func() (*enclavedb.Contract, error) { return enclavedb.ReadContractByAddress(ctx, dbTX, addr) }) } func (es *eventsStorage) ReadContract(ctx context.Context, addr gethcommon.Address) (*enclavedb.Contract, error) { defer es.logDuration("readContract", measure.NewStopwatch()) - return es.cachingService.ReadContractAddr(ctx, addr, func(v any) (*enclavedb.Contract, error) { + return es.cachingService.ReadContractAddr(ctx, addr, func() (*enclavedb.Contract, error) { dbtx, err := es.db.GetSQLDB().BeginTx(ctx, nil) if err != nil { return nil, err @@ -338,14 +338,14 @@ func (es *eventsStorage) ReadContract(ctx context.Context, addr gethcommon.Addre func (es *eventsStorage) findTopic(ctx context.Context, dbTX *sql.Tx, topic []byte, eventTypeId uint64) (*enclavedb.EventTopic, error) { defer es.logDuration("findTopic", measure.NewStopwatch()) - return es.cachingService.ReadEventTopic(ctx, topic, eventTypeId, func(v any) (*enclavedb.EventTopic, error) { + return es.cachingService.ReadEventTopic(ctx, topic, eventTypeId, func() (*enclavedb.EventTopic, error) { return enclavedb.ReadEventTopic(ctx, dbTX, topic, eventTypeId) }) } func (es *eventsStorage) readEOA(ctx context.Context, dbTX *sql.Tx, addr gethcommon.Address) (*uint64, error) { defer es.logDuration("ReadEOA", measure.NewStopwatch()) - return es.cachingService.ReadEOA(ctx, addr, func(v any) (*uint64, error) { + return es.cachingService.ReadEOA(ctx, addr, func() (*uint64, error) { id, err := enclavedb.ReadEoa(ctx, dbTX, addr) if err != nil { return nil, err diff --git a/go/enclave/storage/storage.go b/go/enclave/storage/storage.go index 753a03878..3131e75bc 100644 --- a/go/enclave/storage/storage.go +++ b/go/enclave/storage/storage.go @@ -143,8 +143,8 @@ func (s *storageImpl) FetchBatch(ctx context.Context, hash common.L2BatchHash) ( } func (s *storageImpl) fetchSeqNoByHash(ctx context.Context, hash common.L2BatchHash) (*big.Int, error) { - seqNo, err := s.cachingService.ReadBatchSeqByHash(ctx, hash, func(v any) (*big.Int, error) { - batch, err := enclavedb.ReadBatchHeaderByHash(ctx, s.db.GetSQLDB(), v.(common.L2BatchHash)) + seqNo, err := s.cachingService.ReadBatchSeqByHash(ctx, hash, func() (*big.Int, error) { + batch, err := enclavedb.ReadBatchHeaderByHash(ctx, s.db.GetSQLDB(), hash) if err != nil { return nil, err } @@ -160,7 +160,7 @@ func (s *storageImpl) FetchConvertedHash(ctx context.Context, hash common.L2Batc return gethcommon.Hash{}, err } - convertedHash, err := s.cachingService.ReadConvertedHash(ctx, hash, func(v any) (*gethcommon.Hash, error) { + convertedHash, err := s.cachingService.ReadConvertedHash(ctx, hash, func() (*gethcommon.Hash, error) { ch, err := enclavedb.FetchConvertedBatchHash(ctx, s.db.GetSQLDB(), batch.SequencerOrderNo.Uint64()) if err != nil { return nil, err @@ -185,7 +185,7 @@ func (s *storageImpl) FetchBatchHeader(ctx context.Context, hash common.L2BatchH func (s *storageImpl) FetchBatchTransactionsBySeq(ctx context.Context, seqNo uint64) ([]*common.L2Tx, error) { defer s.logDuration("FetchBatchTransactionsBySeq", measure.NewStopwatch()) - batch, err := s.cachingService.ReadBatch(ctx, seqNo, func(_ any) (*core.Batch, error) { + batch, err := s.cachingService.ReadBatch(ctx, seqNo, func() (*core.Batch, error) { batchHeader, err := s.FetchBatchHeaderBySeqNo(ctx, seqNo) if err != nil { return nil, err @@ -204,7 +204,7 @@ func (s *storageImpl) FetchBatchTransactionsBySeq(ctx context.Context, seqNo uin func (s *storageImpl) FetchBatchByHeight(ctx context.Context, height uint64) (*core.Batch, error) { defer s.logDuration("FetchBatchByHeight", measure.NewStopwatch()) - seqNo, err := s.cachingService.ReadBatchSeqByHeight(ctx, height, func(h any) (*big.Int, error) { + seqNo, err := s.cachingService.ReadBatchSeqByHeight(ctx, height, func() (*big.Int, error) { batch, err := enclavedb.ReadCanonicalBatchHeaderByHeight(ctx, s.db.GetSQLDB(), height) if err != nil { return nil, err @@ -304,8 +304,8 @@ func (s *storageImpl) StoreBlock(ctx context.Context, block *types.Header, chain func (s *storageImpl) FetchBlock(ctx context.Context, blockHash common.L1BlockHash) (*types.Header, error) { defer s.logDuration("FetchBlockHeader", measure.NewStopwatch()) - return s.cachingService.ReadBlock(ctx, blockHash, func(hash any) (*types.Header, error) { - return enclavedb.FetchBlockHeader(ctx, s.db.GetSQLDB(), hash.(common.L1BlockHash)) + return s.cachingService.ReadBlock(ctx, blockHash, func() (*types.Header, error) { + return enclavedb.FetchBlockHeader(ctx, s.db.GetSQLDB(), blockHash) }) } @@ -453,7 +453,7 @@ func (s *storageImpl) ExistsTransactionReceipt(ctx context.Context, txHash commo func (s *storageImpl) GetEnclavePubKey(ctx context.Context, enclaveId common.EnclaveID) (*AttestedEnclave, error) { defer s.logDuration("GetEnclavePubKey", measure.NewStopwatch()) - return s.cachingService.ReadEnclavePubKey(ctx, enclaveId, func(a any) (*AttestedEnclave, error) { + return s.cachingService.ReadEnclavePubKey(ctx, enclaveId, func() (*AttestedEnclave, error) { key, nodeType, err := enclavedb.FetchAttestation(ctx, s.db.GetSQLDB(), enclaveId) if err != nil { return nil, fmt.Errorf("could not retrieve attestation key for address %s. Cause: %w", enclaveId, err) @@ -527,7 +527,7 @@ func (s *storageImpl) FetchBatchBySeqNo(ctx context.Context, seqNum uint64) (*co func (s *storageImpl) FetchBatchHeaderBySeqNo(ctx context.Context, seqNum uint64) (*common.BatchHeader, error) { defer s.logDuration("FetchBatchHeaderBySeqNo", measure.NewStopwatch()) - return s.cachingService.ReadBatchHeader(ctx, seqNum, func(seq any) (*common.BatchHeader, error) { + return s.cachingService.ReadBatchHeader(ctx, seqNum, func() (*common.BatchHeader, error) { return enclavedb.ReadBatchHeaderBySeqNo(ctx, s.db.GetSQLDB(), seqNum) }) } @@ -855,7 +855,7 @@ func (s *storageImpl) CountTransactionsPerAddress(ctx context.Context, address * func (s *storageImpl) readOrWriteEOA(ctx context.Context, dbTX *sql.Tx, addr gethcommon.Address) (*uint64, error) { defer s.logDuration("readOrWriteEOA", measure.NewStopwatch()) - return s.cachingService.ReadEOA(ctx, addr, func(v any) (*uint64, error) { + return s.cachingService.ReadEOA(ctx, addr, func() (*uint64, error) { id, err := enclavedb.ReadEoa(ctx, dbTX, addr) if err != nil { if errors.Is(err, errutil.ErrNotFound) { @@ -928,15 +928,15 @@ func (s *storageImpl) GetSystemContractAddresses(ctx context.Context) (common.Sy func (s *storageImpl) GetSequencerEnclaveIDs(ctx context.Context) ([]common.EnclaveID, error) { defer s.logDuration("GetSequencerEnclaveIDs", measure.NewStopwatch()) - ids, err := s.cachingService.ReadSequencerIDs(ctx, func(any) (*[]common.EnclaveID, error) { + ids, err := s.cachingService.ReadSequencerIDs(ctx, func() ([]common.EnclaveID, error) { sequencerIDs, err := enclavedb.FetchSequencerEnclaveIDs(ctx, s.db.GetSQLDB()) if err != nil { return nil, fmt.Errorf("failed to fetch sequencer IDs from database. Cause: %w", err) } - return &sequencerIDs, nil + return sequencerIDs, nil }) if err != nil { return nil, fmt.Errorf("failed to read sequencer IDs from cache. Cause: %w", err) } - return *ids, nil + return ids, nil } diff --git a/tools/walletextension/cache/RistrettoCache.go b/tools/walletextension/cache/RistrettoCache.go index ab1a492cd..a96fc9ce5 100644 --- a/tools/walletextension/cache/RistrettoCache.go +++ b/tools/walletextension/cache/RistrettoCache.go @@ -7,7 +7,7 @@ import ( "github.com/ethereum/go-ethereum/log" - "github.com/dgraph-io/ristretto" + "github.com/dgraph-io/ristretto/v2" ) const ( @@ -16,7 +16,7 @@ const ( ) type ristrettoCache struct { - cache *ristretto.Cache + cache *ristretto.Cache[[]byte, any] quit chan struct{} lastEviction time.Time shortLivingEnabled *atomic.Bool @@ -24,7 +24,7 @@ type ristrettoCache struct { // NewRistrettoCacheWithEviction returns a new ristrettoCache. func NewRistrettoCacheWithEviction(nrElems int, logger log.Logger) (Cache, error) { - cache, err := ristretto.NewCache(&ristretto.Config{ + cache, err := ristretto.NewCache[[]byte, any](&ristretto.Config[[]byte, any]{ NumCounters: int64(nrElems * 10), MaxCost: int64(nrElems), BufferItems: bufferItems, @@ -58,7 +58,7 @@ func (c *ristrettoCache) DisableShortLiving() { c.shortLivingEnabled.Store(false) } -func (c *ristrettoCache) IsEvicted(key any, originalTTL time.Duration) bool { +func (c *ristrettoCache) IsEvicted(key []byte, originalTTL time.Duration) bool { if !c.shortLivingEnabled.Load() { return true } diff --git a/tools/walletextension/cache/cache.go b/tools/walletextension/cache/cache.go index 9043351fa..ae82f905f 100644 --- a/tools/walletextension/cache/cache.go +++ b/tools/walletextension/cache/cache.go @@ -15,7 +15,7 @@ type Cache interface { DisableShortLiving() // IsEvicted - based on the eviction event and the time of caching, calculates whether the key was evicted - IsEvicted(key any, originalTTL time.Duration) bool + IsEvicted(key []byte, originalTTL time.Duration) bool Set(key []byte, value any, ttl time.Duration) bool Get(key []byte) (value any, ok bool) From 817626f5ee0bef22b953ab49f716c5000c1c05f7 Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Mon, 23 Dec 2024 17:33:42 +0200 Subject: [PATCH 02/11] rework cache --- go/enclave/storage/cache_service.go | 102 ++++++++++++--------------- go/enclave/storage/events_storage.go | 5 -- go/enclave/storage/storage.go | 11 +++ 3 files changed, 57 insertions(+), 61 deletions(-) diff --git a/go/enclave/storage/cache_service.go b/go/enclave/storage/cache_service.go index e4775df24..9f44785d2 100644 --- a/go/enclave/storage/cache_service.go +++ b/go/enclave/storage/cache_service.go @@ -20,18 +20,8 @@ import ( "github.com/ten-protocol/go-ten/go/common/log" ) -// approximate cost in bytes of the cached values const ( - blockHeaderCost = 1024 - batchHeaderCost = 1024 - hashCost = 32 - idCost = 8 - batchCost = 1024 * 1024 - receiptCost = 1024 * 50 - contractCost = 60 - eventTypeCost = 120 - eventTopicCost = 16 - enclaveCost = 100 + cacheCost = 1 ) type CacheService struct { @@ -90,34 +80,34 @@ func NewCacheService(logger gethlog.Logger, testMode bool) *CacheService { nrContractAddresses := 10_000 // ~1M nrBatchesWithContent := 50 // ~100M - nrReceipts := 10_000 // ~1G nrEnclaves := 20 + nrReceipts := 15_000 // ~100M if testMode { - nrReceipts = 500 //~50M + nrReceipts = 500 } return &CacheService{ - blockCache: newCache[[]byte, *types.Header](logger, nrL1Blocks, blockHeaderCost), + blockCache: newCache[[]byte, *types.Header](logger, nrL1Blocks), - batchCacheBySeqNo: newCache[uint64, *common.BatchHeader](logger, nrBatches, batchHeaderCost), - seqCacheByHash: newCache[[]byte, *big.Int](logger, nrBatches, idCost), - seqCacheByHeight: newCache[uint64, *big.Int](logger, nrBatches, idCost), + batchCacheBySeqNo: newCache[uint64, *common.BatchHeader](logger, nrBatches), + seqCacheByHash: newCache[[]byte, *big.Int](logger, nrBatches), + seqCacheByHeight: newCache[uint64, *big.Int](logger, nrBatches), - convertedGethHeaderCache: newCache[[]byte, *types.Header](logger, nrConvertedEth, batchHeaderCost), - convertedHashCache: newCache[[]byte, *gethcommon.Hash](logger, nrConvertedEth, hashCost), + convertedGethHeaderCache: newCache[[]byte, *types.Header](logger, nrConvertedEth), + convertedHashCache: newCache[[]byte, *gethcommon.Hash](logger, nrConvertedEth), - eoaCache: newCache[[]byte, *uint64](logger, nrEOA, idCost), - contractAddressCache: newCache[[]byte, *enclavedb.Contract](logger, nrContractAddresses, contractCost), - eventTypeCache: newCache[[]byte, *enclavedb.EventType](logger, nrEventTypes, eventTypeCost), - eventTopicCache: newCache[[]byte, *enclavedb.EventTopic](logger, nrEventTypes, eventTopicCost), + eoaCache: newCache[[]byte, *uint64](logger, nrEOA), + contractAddressCache: newCache[[]byte, *enclavedb.Contract](logger, nrContractAddresses), + eventTypeCache: newCache[[]byte, *enclavedb.EventType](logger, nrEventTypes), + eventTopicCache: newCache[[]byte, *enclavedb.EventTopic](logger, nrEventTypes), - receiptCache: newCache[[]byte, *CachedReceipt](logger, nrReceipts, receiptCost), - attestedEnclavesCache: newCache[[]byte, *AttestedEnclave](logger, nrEnclaves, enclaveCost), + receiptCache: newCache[[]byte, *CachedReceipt](logger, nrReceipts), + attestedEnclavesCache: newCache[[]byte, *AttestedEnclave](logger, nrEnclaves), // cache the latest received batches to avoid a lookup when streaming it back to the host after processing - lastBatchesCache: newCache[uint64, *core.Batch](logger, nrBatchesWithContent, batchCost), + lastBatchesCache: newCache[uint64, *core.Batch](logger, nrBatchesWithContent), sequencerIDsCache: make([]common.EnclaveID, 0), @@ -125,11 +115,11 @@ func NewCacheService(logger gethlog.Logger, testMode bool) *CacheService { } } -func newCache[K ristretto.Key, V any](logger gethlog.Logger, nrElem, capacityPerElem int) *ristretto.Cache[K, V] { +func newCache[K ristretto.Key, V any](logger gethlog.Logger, nrElem int) *ristretto.Cache[K, V] { ristrettoCache, err := ristretto.NewCache(&ristretto.Config[K, V]{ - NumCounters: int64(10 * nrElem), // 10 times the expected elements - MaxCost: int64(capacityPerElem * nrElem * 2), // calculate the max cost - BufferItems: 64, // number of keys per Get buffer. + NumCounters: int64(10 * nrElem), // 10 times the expected elements + MaxCost: int64(nrElem), // calculate the max cost + BufferItems: 64, // number of keys per Get buffer. Metrics: true, }) if err != nil { @@ -139,66 +129,66 @@ func newCache[K ristretto.Key, V any](logger gethlog.Logger, nrElem, capacityPer } func (cs *CacheService) CacheConvertedHash(ctx context.Context, batchHash, convertedHash gethcommon.Hash) { - cacheValue(ctx, cs.convertedHashCache, cs.logger, batchHash.Bytes(), &convertedHash, hashCost) + cacheValue(ctx, cs.convertedHashCache, cs.logger, batchHash.Bytes(), &convertedHash) } func (cs *CacheService) CacheBlock(ctx context.Context, b *types.Header) { - cacheValue(ctx, cs.blockCache, cs.logger, b.Hash().Bytes(), b, blockHeaderCost) + cacheValue(ctx, cs.blockCache, cs.logger, b.Hash().Bytes(), b) } func (cs *CacheService) CacheReceipt(_ context.Context, r *CachedReceipt) { - // keep the receipts in cache for 1 minute - cs.receiptCache.SetWithTTL(r.Receipt.TxHash.Bytes(), r, receiptCost, 1*time.Minute) + // keep the receipts in cache for 100 seconds - 100 batches + cs.receiptCache.SetWithTTL(r.Receipt.TxHash.Bytes(), r, cacheCost, 100*time.Second) } func (cs *CacheService) CacheBatch(ctx context.Context, batch *core.Batch) { - cacheValue(ctx, cs.batchCacheBySeqNo, cs.logger, batch.SeqNo().Uint64(), batch.Header, batchHeaderCost) - cacheValue(ctx, cs.seqCacheByHash, cs.logger, batch.Hash().Bytes(), batch.SeqNo(), idCost) + cacheValue(ctx, cs.batchCacheBySeqNo, cs.logger, batch.SeqNo().Uint64(), batch.Header) + cacheValue(ctx, cs.seqCacheByHash, cs.logger, batch.Hash().Bytes(), batch.SeqNo()) // note: the key is (height+1), because for some reason it doesn't like a key of 0 // should always contain the canonical batch because the cache is overwritten by each new batch after a reorg - cacheValue(ctx, cs.seqCacheByHeight, cs.logger, batch.NumberU64()+1, batch.SeqNo(), idCost) + cacheValue(ctx, cs.seqCacheByHeight, cs.logger, batch.NumberU64()+1, batch.SeqNo()) - cacheValue(ctx, cs.lastBatchesCache, cs.logger, batch.SeqNo().Uint64(), batch, batchCost) + cacheValue(ctx, cs.lastBatchesCache, cs.logger, batch.SeqNo().Uint64(), batch) } func (cs *CacheService) ReadBlock(ctx context.Context, key gethcommon.Hash, onCacheMiss func() (*types.Header, error)) (*types.Header, error) { - return getCachedValue(ctx, cs.blockCache, cs.logger, key.Bytes(), blockHeaderCost, onCacheMiss, true) + return getCachedValue(ctx, cs.blockCache, cs.logger, key.Bytes(), onCacheMiss, true) } func (cs *CacheService) ReadBatchSeqByHash(ctx context.Context, hash common.L2BatchHash, onCacheMiss func() (*big.Int, error)) (*big.Int, error) { - return getCachedValue(ctx, cs.seqCacheByHash, cs.logger, hash.Bytes(), idCost, onCacheMiss, true) + return getCachedValue(ctx, cs.seqCacheByHash, cs.logger, hash.Bytes(), onCacheMiss, true) } func (cs *CacheService) ReadBatchSeqByHeight(ctx context.Context, height uint64, onCacheMiss func() (*big.Int, error)) (*big.Int, error) { // the key is (height+1), because for some reason it doesn't like a key of 0 - return getCachedValue(ctx, cs.seqCacheByHeight, cs.logger, height+1, idCost, onCacheMiss, true) + return getCachedValue(ctx, cs.seqCacheByHeight, cs.logger, height+1, onCacheMiss, true) } func (cs *CacheService) ReadConvertedHash(ctx context.Context, hash common.L2BatchHash, onCacheMiss func() (*gethcommon.Hash, error)) (*gethcommon.Hash, error) { - return getCachedValue(ctx, cs.convertedHashCache, cs.logger, hash.Bytes(), hashCost, onCacheMiss, true) + return getCachedValue(ctx, cs.convertedHashCache, cs.logger, hash.Bytes(), onCacheMiss, true) } func (cs *CacheService) ReadBatchHeader(ctx context.Context, seqNum uint64, onCacheMiss func() (*common.BatchHeader, error)) (*common.BatchHeader, error) { - return getCachedValue(ctx, cs.batchCacheBySeqNo, cs.logger, seqNum, batchHeaderCost, onCacheMiss, true) + return getCachedValue(ctx, cs.batchCacheBySeqNo, cs.logger, seqNum, onCacheMiss, true) } func (cs *CacheService) ReadBatch(ctx context.Context, seqNum uint64, onCacheMiss func() (*core.Batch, error)) (*core.Batch, error) { - return getCachedValue(ctx, cs.lastBatchesCache, cs.logger, seqNum, batchCost, onCacheMiss, true) + return getCachedValue(ctx, cs.lastBatchesCache, cs.logger, seqNum, onCacheMiss, true) } func (cs *CacheService) ReadEOA(ctx context.Context, addr gethcommon.Address, onCacheMiss func() (*uint64, error)) (*uint64, error) { - return getCachedValue(ctx, cs.eoaCache, cs.logger, addr.Bytes(), idCost, onCacheMiss, true) + return getCachedValue(ctx, cs.eoaCache, cs.logger, addr.Bytes(), onCacheMiss, true) } func (cs *CacheService) ReadContractAddr(ctx context.Context, addr gethcommon.Address, onCacheMiss func() (*enclavedb.Contract, error)) (*enclavedb.Contract, error) { - return getCachedValue(ctx, cs.contractAddressCache, cs.logger, addr.Bytes(), contractCost, onCacheMiss, true) + return getCachedValue(ctx, cs.contractAddressCache, cs.logger, addr.Bytes(), onCacheMiss, true) } func (cs *CacheService) ReadEventTopic(ctx context.Context, topic []byte, eventTypeId uint64, onCacheMiss func() (*enclavedb.EventTopic, error)) (*enclavedb.EventTopic, error) { b := make([]byte, 8) binary.LittleEndian.PutUint64(b, eventTypeId) key := append(topic, b...) - return getCachedValue(ctx, cs.eventTopicCache, cs.logger, key, eventTopicCost, onCacheMiss, true) + return getCachedValue(ctx, cs.eventTopicCache, cs.logger, key, onCacheMiss, true) } type CachedReceipt struct { @@ -208,7 +198,7 @@ type CachedReceipt struct { } func (cs *CacheService) ReadReceipt(ctx context.Context, txHash gethcommon.Hash) (*CachedReceipt, error) { - return getCachedValue(ctx, cs.receiptCache, cs.logger, txHash.Bytes(), receiptCost, nil, false) + return getCachedValue(ctx, cs.receiptCache, cs.logger, txHash.Bytes(), nil, false) } func (cs *CacheService) DelReceipt(_ context.Context, txHash gethcommon.Hash) error { @@ -220,15 +210,15 @@ func (cs *CacheService) ReadEventType(ctx context.Context, contractAddress gethc key := make([]byte, 0) key = append(key, contractAddress.Bytes()...) key = append(key, eventSignature.Bytes()...) - return getCachedValue(ctx, cs.eventTypeCache, cs.logger, key, eventTypeCost, onCacheMiss, true) + return getCachedValue(ctx, cs.eventTypeCache, cs.logger, key, onCacheMiss, true) } func (cs *CacheService) ReadConvertedHeader(ctx context.Context, batchHash common.L2BatchHash, onCacheMiss func() (*types.Header, error)) (*types.Header, error) { - return getCachedValue(ctx, cs.convertedGethHeaderCache, cs.logger, batchHash.Bytes(), blockHeaderCost, onCacheMiss, true) + return getCachedValue(ctx, cs.convertedGethHeaderCache, cs.logger, batchHash.Bytes(), onCacheMiss, true) } func (cs *CacheService) ReadEnclavePubKey(ctx context.Context, enclaveId common.EnclaveID, onCacheMiss func() (*AttestedEnclave, error)) (*AttestedEnclave, error) { - return getCachedValue(ctx, cs.attestedEnclavesCache, cs.logger, enclaveId.Bytes(), enclaveCost, onCacheMiss, true) + return getCachedValue(ctx, cs.attestedEnclavesCache, cs.logger, enclaveId.Bytes(), onCacheMiss, true) } func (cs *CacheService) UpdateEnclaveNodeType(ctx context.Context, enclaveId common.EnclaveID, nodeType common.NodeType) { @@ -238,7 +228,7 @@ func (cs *CacheService) UpdateEnclaveNodeType(ctx context.Context, enclaveId com return } enclave.Type = nodeType - cacheValue(ctx, cs.attestedEnclavesCache, cs.logger, enclaveId.Bytes(), enclave, enclaveCost) + cacheValue(ctx, cs.attestedEnclavesCache, cs.logger, enclaveId.Bytes(), enclave) } func (cs *CacheService) CacheSequencerIDs(_ context.Context, sequencerIDs []common.EnclaveID) { @@ -258,7 +248,7 @@ func (cs *CacheService) ReadSequencerIDs(_ context.Context, onCacheMiss func() ( // getCachedValue - returns the cached value for the provided key. If the key is not found, then invoke the 'onCacheMiss' function // which returns the value, and cache it -func getCachedValue[K ristretto.Key, V any](ctx context.Context, cache *ristretto.Cache[K, V], logger gethlog.Logger, key K, cost int64, onCacheMiss func() (V, error), cacheIfMissing bool) (V, error) { +func getCachedValue[K ristretto.Key, V any](ctx context.Context, cache *ristretto.Cache[K, V], logger gethlog.Logger, key K, onCacheMiss func() (V, error), cacheIfMissing bool) (V, error) { value, found := cache.Get(key) if found { return value, nil @@ -272,13 +262,13 @@ func getCachedValue[K ristretto.Key, V any](ctx context.Context, cache *ristrett return v, err } if cacheIfMissing { - cacheValue(ctx, cache, logger, key, v, cost) + cacheValue(ctx, cache, logger, key, v) } return v, nil } -func cacheValue[K ristretto.Key, V any](_ context.Context, cache *ristretto.Cache[K, V], logger gethlog.Logger, key K, v V, cost int64) { - added := cache.Set(key, v, cost) +func cacheValue[K ristretto.Key, V any](_ context.Context, cache *ristretto.Cache[K, V], logger gethlog.Logger, key K, v V) { + added := cache.Set(key, v, cacheCost) if !added { logger.Debug("Did not store value in cache") } diff --git a/go/enclave/storage/events_storage.go b/go/enclave/storage/events_storage.go index 0d4df41ff..688561b74 100644 --- a/go/enclave/storage/events_storage.go +++ b/go/enclave/storage/events_storage.go @@ -53,11 +53,6 @@ func (es *eventsStorage) storeReceiptAndEventLogs(ctx context.Context, dbTX *sql } } - es.cachingService.CacheReceipt(ctx, &CachedReceipt{ - Receipt: txExecResult.Receipt, - From: txExecResult.TxWithSender.Sender, - To: txExecResult.TxWithSender.Tx.To(), - }) return nil } diff --git a/go/enclave/storage/storage.go b/go/enclave/storage/storage.go index 3131e75bc..a58d75b35 100644 --- a/go/enclave/storage/storage.go +++ b/go/enclave/storage/storage.go @@ -693,6 +693,17 @@ func (s *storageImpl) StoreExecutedBatch(ctx context.Context, batch *core.Batch, return fmt.Errorf("could not commit batch %w", err) } + // after a successful db commit, cache the receipts + if s.config.StoreExecutedTransactions { + for _, txExecResult := range results { + s.cachingService.CacheReceipt(ctx, &CachedReceipt{ + Receipt: txExecResult.Receipt, + From: txExecResult.TxWithSender.Sender, + To: txExecResult.TxWithSender.Tx.To(), + }) + } + } + return nil } From 8bb357590780522a0fb9d8b2c480816945c66be3 Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Tue, 24 Dec 2024 12:22:27 +0200 Subject: [PATCH 03/11] add predictable FIFO cache for receipts and last batches --- go.mod | 1 + go.sum | 2 + go/enclave/storage/cache_service.go | 87 +++++++++++++++++++++-------- go/enclave/storage/storage.go | 9 +-- 4 files changed, 68 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index 2cabde496..099944eea 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos v1.1.0 github.com/FantasyJony/openzeppelin-merkle-tree-go v1.1.3 github.com/Microsoft/go-winio v0.6.2 + github.com/TwiN/gocache/v2 v2.2.2 github.com/andybalholm/brotli v1.1.1 github.com/codeclysm/extract/v3 v3.1.1 github.com/deckarep/golang-set/v2 v2.6.0 diff --git a/go.sum b/go.sum index b5cbd7104..f77e511b1 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/FantasyJony/openzeppelin-merkle-tree-go v1.1.3 h1:KzMvCFet0baw6uJnxTE github.com/FantasyJony/openzeppelin-merkle-tree-go v1.1.3/go.mod h1:OiwyYqbtMkQH+VzA4b8lI+qHnExJy0fIdz+59/8nFes= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/TwiN/gocache/v2 v2.2.2 h1:4HToPfDV8FSbaYO5kkbhLpEllUYse5rAf+hVU/mSsuI= +github.com/TwiN/gocache/v2 v2.2.2/go.mod h1:WfIuwd7GR82/7EfQqEtmLFC3a2vqaKbs4Pe6neB7Gyc= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= diff --git a/go/enclave/storage/cache_service.go b/go/enclave/storage/cache_service.go index 9f44785d2..bba82f87d 100644 --- a/go/enclave/storage/cache_service.go +++ b/go/enclave/storage/cache_service.go @@ -3,6 +3,7 @@ package storage import ( "context" "encoding/binary" + "fmt" "math/big" "time" @@ -12,6 +13,7 @@ import ( "github.com/ten-protocol/go-ten/go/enclave/core" + "github.com/TwiN/gocache/v2" "github.com/dgraph-io/ristretto/v2" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -54,12 +56,12 @@ type CacheService struct { eventTypeCache *ristretto.Cache[[]byte, *enclavedb.EventType] // store the last few batches together with the content - lastBatchesCache *ristretto.Cache[uint64, *core.Batch] + lastBatchesCache *gocache.Cache // store all recent receipts in a cache // together with the sender - and for each log whether it is visible by the sender // only sender can view configured - receiptCache *ristretto.Cache[[]byte, *CachedReceipt] + receiptCache *gocache.Cache // store the enclaves from the network attestedEnclavesCache *ristretto.Cache[[]byte, *AttestedEnclave] @@ -89,25 +91,25 @@ func NewCacheService(logger gethlog.Logger, testMode bool) *CacheService { } return &CacheService{ - blockCache: newCache[[]byte, *types.Header](logger, nrL1Blocks), + blockCache: newLFUCache[[]byte, *types.Header](logger, nrL1Blocks), - batchCacheBySeqNo: newCache[uint64, *common.BatchHeader](logger, nrBatches), - seqCacheByHash: newCache[[]byte, *big.Int](logger, nrBatches), - seqCacheByHeight: newCache[uint64, *big.Int](logger, nrBatches), + batchCacheBySeqNo: newLFUCache[uint64, *common.BatchHeader](logger, nrBatches), + seqCacheByHash: newLFUCache[[]byte, *big.Int](logger, nrBatches), + seqCacheByHeight: newLFUCache[uint64, *big.Int](logger, nrBatches), - convertedGethHeaderCache: newCache[[]byte, *types.Header](logger, nrConvertedEth), - convertedHashCache: newCache[[]byte, *gethcommon.Hash](logger, nrConvertedEth), + convertedGethHeaderCache: newLFUCache[[]byte, *types.Header](logger, nrConvertedEth), + convertedHashCache: newLFUCache[[]byte, *gethcommon.Hash](logger, nrConvertedEth), - eoaCache: newCache[[]byte, *uint64](logger, nrEOA), - contractAddressCache: newCache[[]byte, *enclavedb.Contract](logger, nrContractAddresses), - eventTypeCache: newCache[[]byte, *enclavedb.EventType](logger, nrEventTypes), - eventTopicCache: newCache[[]byte, *enclavedb.EventTopic](logger, nrEventTypes), + eoaCache: newLFUCache[[]byte, *uint64](logger, nrEOA), + contractAddressCache: newLFUCache[[]byte, *enclavedb.Contract](logger, nrContractAddresses), + eventTypeCache: newLFUCache[[]byte, *enclavedb.EventType](logger, nrEventTypes), + eventTopicCache: newLFUCache[[]byte, *enclavedb.EventTopic](logger, nrEventTypes), - receiptCache: newCache[[]byte, *CachedReceipt](logger, nrReceipts), - attestedEnclavesCache: newCache[[]byte, *AttestedEnclave](logger, nrEnclaves), + receiptCache: newFifoCache(nrReceipts, 100*time.Second), + attestedEnclavesCache: newLFUCache[[]byte, *AttestedEnclave](logger, nrEnclaves), // cache the latest received batches to avoid a lookup when streaming it back to the host after processing - lastBatchesCache: newCache[uint64, *core.Batch](logger, nrBatchesWithContent), + lastBatchesCache: newFifoCache(nrBatchesWithContent, time.Duration(nrBatchesWithContent)*time.Second), sequencerIDsCache: make([]common.EnclaveID, 0), @@ -115,7 +117,12 @@ func NewCacheService(logger gethlog.Logger, testMode bool) *CacheService { } } -func newCache[K ristretto.Key, V any](logger gethlog.Logger, nrElem int) *ristretto.Cache[K, V] { +func (cs *CacheService) Stop() { + cs.receiptCache.StopJanitor() + cs.lastBatchesCache.StopJanitor() +} + +func newLFUCache[K ristretto.Key, V any](logger gethlog.Logger, nrElem int) *ristretto.Cache[K, V] { ristrettoCache, err := ristretto.NewCache(&ristretto.Config[K, V]{ NumCounters: int64(10 * nrElem), // 10 times the expected elements MaxCost: int64(nrElem), // calculate the max cost @@ -128,6 +135,15 @@ func newCache[K ristretto.Key, V any](logger gethlog.Logger, nrElem int) *ristre return ristrettoCache } +func newFifoCache(nrElem int, ttl time.Duration) *gocache.Cache { + cache := gocache.NewCache().WithMaxSize(nrElem).WithEvictionPolicy(gocache.FirstInFirstOut).WithDefaultTTL(ttl) + err := cache.StartJanitor() + if err != nil { + panic("failed to start cache.") + } + return cache +} + func (cs *CacheService) CacheConvertedHash(ctx context.Context, batchHash, convertedHash gethcommon.Hash) { cacheValue(ctx, cs.convertedHashCache, cs.logger, batchHash.Bytes(), &convertedHash) } @@ -136,9 +152,16 @@ func (cs *CacheService) CacheBlock(ctx context.Context, b *types.Header) { cacheValue(ctx, cs.blockCache, cs.logger, b.Hash().Bytes(), b) } -func (cs *CacheService) CacheReceipt(_ context.Context, r *CachedReceipt) { - // keep the receipts in cache for 100 seconds - 100 batches - cs.receiptCache.SetWithTTL(r.Receipt.TxHash.Bytes(), r, cacheCost, 100*time.Second) +func (cs *CacheService) CacheReceipts(results core.TxExecResults) { + receipts := make(map[string]any) + for _, txExecResult := range results { + receipts[txExecResult.TxWithSender.Tx.Hash().String()] = &CachedReceipt{ + Receipt: txExecResult.Receipt, + From: txExecResult.TxWithSender.Sender, + To: txExecResult.TxWithSender.Tx.To(), + } + } + cs.receiptCache.SetAll(receipts) } func (cs *CacheService) CacheBatch(ctx context.Context, batch *core.Batch) { @@ -148,7 +171,7 @@ func (cs *CacheService) CacheBatch(ctx context.Context, batch *core.Batch) { // should always contain the canonical batch because the cache is overwritten by each new batch after a reorg cacheValue(ctx, cs.seqCacheByHeight, cs.logger, batch.NumberU64()+1, batch.SeqNo()) - cacheValue(ctx, cs.lastBatchesCache, cs.logger, batch.SeqNo().Uint64(), batch) + cs.lastBatchesCache.Set(batch.SeqNo().String(), batch) } func (cs *CacheService) ReadBlock(ctx context.Context, key gethcommon.Hash, onCacheMiss func() (*types.Header, error)) (*types.Header, error) { @@ -173,7 +196,15 @@ func (cs *CacheService) ReadBatchHeader(ctx context.Context, seqNum uint64, onCa } func (cs *CacheService) ReadBatch(ctx context.Context, seqNum uint64, onCacheMiss func() (*core.Batch, error)) (*core.Batch, error) { - return getCachedValue(ctx, cs.lastBatchesCache, cs.logger, seqNum, onCacheMiss, true) + b, found := cs.lastBatchesCache.Get(fmt.Sprintf("%d", seqNum)) + if !found { + return nil, errutil.ErrNotFound + } + cb, ok := b.(*core.Batch) + if !ok { + return nil, fmt.Errorf("should not happen. invalid cached batch") + } + return cb, nil } func (cs *CacheService) ReadEOA(ctx context.Context, addr gethcommon.Address, onCacheMiss func() (*uint64, error)) (*uint64, error) { @@ -197,12 +228,20 @@ type CachedReceipt struct { To *gethcommon.Address } -func (cs *CacheService) ReadReceipt(ctx context.Context, txHash gethcommon.Hash) (*CachedReceipt, error) { - return getCachedValue(ctx, cs.receiptCache, cs.logger, txHash.Bytes(), nil, false) +func (cs *CacheService) ReadReceipt(_ context.Context, txHash gethcommon.Hash) (*CachedReceipt, error) { + r, found := cs.receiptCache.Get(txHash.String()) + if !found { + return nil, errutil.ErrNotFound + } + cr, ok := r.(*CachedReceipt) + if !ok { + return nil, fmt.Errorf("should not happen. invalid cached receipt") + } + return cr, nil } func (cs *CacheService) DelReceipt(_ context.Context, txHash gethcommon.Hash) error { - cs.receiptCache.Del(txHash.Bytes()) + cs.receiptCache.Delete(txHash.String()) return nil } diff --git a/go/enclave/storage/storage.go b/go/enclave/storage/storage.go index a58d75b35..04728f7ce 100644 --- a/go/enclave/storage/storage.go +++ b/go/enclave/storage/storage.go @@ -116,6 +116,7 @@ func (s *storageImpl) StateDB() state.Database { } func (s *storageImpl) Close() error { + s.cachingService.Stop() return s.db.GetSQLDB().Close() } @@ -695,13 +696,7 @@ func (s *storageImpl) StoreExecutedBatch(ctx context.Context, batch *core.Batch, // after a successful db commit, cache the receipts if s.config.StoreExecutedTransactions { - for _, txExecResult := range results { - s.cachingService.CacheReceipt(ctx, &CachedReceipt{ - Receipt: txExecResult.Receipt, - From: txExecResult.TxWithSender.Sender, - To: txExecResult.TxWithSender.Tx.To(), - }) - } + s.cachingService.CacheReceipts(results) } return nil From b25ad0e1b6f7bbce3456e80d14efdf5b11bf7fef Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Fri, 27 Dec 2024 12:13:59 +0200 Subject: [PATCH 04/11] fixes --- go/enclave/storage/cache_service.go | 15 ++++++++++----- go/enclave/storage/enclavedb/block.go | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/go/enclave/storage/cache_service.go b/go/enclave/storage/cache_service.go index bba82f87d..5557fab8b 100644 --- a/go/enclave/storage/cache_service.go +++ b/go/enclave/storage/cache_service.go @@ -87,7 +87,7 @@ func NewCacheService(logger gethlog.Logger, testMode bool) *CacheService { nrReceipts := 15_000 // ~100M if testMode { - nrReceipts = 500 + nrReceipts = 2500 } return &CacheService{ @@ -105,11 +105,11 @@ func NewCacheService(logger gethlog.Logger, testMode bool) *CacheService { eventTypeCache: newLFUCache[[]byte, *enclavedb.EventType](logger, nrEventTypes), eventTopicCache: newLFUCache[[]byte, *enclavedb.EventTopic](logger, nrEventTypes), - receiptCache: newFifoCache(nrReceipts, 100*time.Second), + receiptCache: newFifoCache(nrReceipts, 150*time.Second), attestedEnclavesCache: newLFUCache[[]byte, *AttestedEnclave](logger, nrEnclaves), // cache the latest received batches to avoid a lookup when streaming it back to the host after processing - lastBatchesCache: newFifoCache(nrBatchesWithContent, time.Duration(nrBatchesWithContent)*time.Second), + lastBatchesCache: newFifoCache(nrBatchesWithContent, gocache.NoExpiration), sequencerIDsCache: make([]common.EnclaveID, 0), @@ -171,7 +171,7 @@ func (cs *CacheService) CacheBatch(ctx context.Context, batch *core.Batch) { // should always contain the canonical batch because the cache is overwritten by each new batch after a reorg cacheValue(ctx, cs.seqCacheByHeight, cs.logger, batch.NumberU64()+1, batch.SeqNo()) - cs.lastBatchesCache.Set(batch.SeqNo().String(), batch) + cs.lastBatchesCache.Set(fmt.Sprintf("%d", batch.SeqNo().Uint64()), batch) } func (cs *CacheService) ReadBlock(ctx context.Context, key gethcommon.Hash, onCacheMiss func() (*types.Header, error)) (*types.Header, error) { @@ -198,7 +198,12 @@ func (cs *CacheService) ReadBatchHeader(ctx context.Context, seqNum uint64, onCa func (cs *CacheService) ReadBatch(ctx context.Context, seqNum uint64, onCacheMiss func() (*core.Batch, error)) (*core.Batch, error) { b, found := cs.lastBatchesCache.Get(fmt.Sprintf("%d", seqNum)) if !found { - return nil, errutil.ErrNotFound + b1, err := onCacheMiss() + if err != nil { + return nil, err + } + cs.CacheBatch(ctx, b1) + b = b1 } cb, ok := b.(*core.Batch) if !ok { diff --git a/go/enclave/storage/enclavedb/block.go b/go/enclave/storage/enclavedb/block.go index b56e05e6e..c7e594329 100644 --- a/go/enclave/storage/enclavedb/block.go +++ b/go/enclave/storage/enclavedb/block.go @@ -80,7 +80,7 @@ func CheckCanonicalValidity(ctx context.Context, dbtx *sql.Tx, blockId int64) er return nil } -// HandleBlockArrivedAfterBatches- handle the corner case where the block wasn't available when the batch was received +// HandleBlockArrivedAfterBatches - handle the corner case where the block wasn't available when the batch was received func HandleBlockArrivedAfterBatches(ctx context.Context, dbtx *sql.Tx, blockId int64, blockHash common.L1BlockHash) error { _, err := dbtx.ExecContext(ctx, "update batch set l1_proof=?, is_canonical=true where l1_proof_hash=?", blockId, blockHash.Bytes()) return err From 4af9b10b57c3aee3704de7027d9d7aed49df36c6 Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Fri, 27 Dec 2024 13:45:52 +0200 Subject: [PATCH 05/11] fixes --- go/enclave/rpc/GetTransactionReceipt.go | 3 ++- go/enclave/storage/cache_service.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go/enclave/rpc/GetTransactionReceipt.go b/go/enclave/rpc/GetTransactionReceipt.go index f051a5d76..aff57eb11 100644 --- a/go/enclave/rpc/GetTransactionReceipt.go +++ b/go/enclave/rpc/GetTransactionReceipt.go @@ -48,11 +48,12 @@ func GetTransactionReceiptExecute(builder *CallBuilder[gethcommon.Hash, map[stri } if result != nil { - rpc.logger.Trace("Successfully retrieved receipt from cache for ", log.TxKey, txHash, "rec", result) + rpc.logger.Info("Cache hit for receipt", log.TxKey, txHash) builder.ReturnValue = &result return nil } + rpc.logger.Info("Cache miss for receipt", log.TxKey, txHash.String()) exists, err := rpc.storage.ExistsTransactionReceipt(builder.ctx, txHash) if err != nil { return fmt.Errorf("could not retrieve transaction receipt in eth_getTransactionReceipt request. Cause: %w", err) diff --git a/go/enclave/storage/cache_service.go b/go/enclave/storage/cache_service.go index 5557fab8b..e7a10bb95 100644 --- a/go/enclave/storage/cache_service.go +++ b/go/enclave/storage/cache_service.go @@ -105,7 +105,8 @@ func NewCacheService(logger gethlog.Logger, testMode bool) *CacheService { eventTypeCache: newLFUCache[[]byte, *enclavedb.EventType](logger, nrEventTypes), eventTopicCache: newLFUCache[[]byte, *enclavedb.EventTopic](logger, nrEventTypes), - receiptCache: newFifoCache(nrReceipts, 150*time.Second), + //receiptCache: newFifoCache(nrReceipts, 150*time.Second), + receiptCache: newFifoCache(nrReceipts, gocache.NoExpiration), attestedEnclavesCache: newLFUCache[[]byte, *AttestedEnclave](logger, nrEnclaves), // cache the latest received batches to avoid a lookup when streaming it back to the host after processing From 9c7364409f3ab84ded09209c48db475562feb7b7 Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Fri, 27 Dec 2024 14:06:37 +0200 Subject: [PATCH 06/11] fixes --- go/enclave/rpc/GetTransactionReceipt.go | 12 ++++++++++++ go/enclave/storage/cache_service.go | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/go/enclave/rpc/GetTransactionReceipt.go b/go/enclave/rpc/GetTransactionReceipt.go index aff57eb11..103922f9a 100644 --- a/go/enclave/rpc/GetTransactionReceipt.go +++ b/go/enclave/rpc/GetTransactionReceipt.go @@ -63,6 +63,18 @@ func GetTransactionReceiptExecute(builder *CallBuilder[gethcommon.Hash, map[stri return nil } + // try the cache again in case the tx was commited in the meantime + result, err = fetchFromCache(builder.ctx, rpc.storage, rpc.cacheService, txHash, requester) + if err != nil { + return err + } + + if result != nil { + rpc.logger.Info("Cache hit for receipt", log.TxKey, txHash) + builder.ReturnValue = &result + return nil + } + // We retrieve the transaction receipt. receipt, err := rpc.storage.GetFilteredInternalReceipt(builder.ctx, txHash, requester, false) if err != nil { diff --git a/go/enclave/storage/cache_service.go b/go/enclave/storage/cache_service.go index e7a10bb95..b49d096a1 100644 --- a/go/enclave/storage/cache_service.go +++ b/go/enclave/storage/cache_service.go @@ -105,7 +105,7 @@ func NewCacheService(logger gethlog.Logger, testMode bool) *CacheService { eventTypeCache: newLFUCache[[]byte, *enclavedb.EventType](logger, nrEventTypes), eventTopicCache: newLFUCache[[]byte, *enclavedb.EventTopic](logger, nrEventTypes), - //receiptCache: newFifoCache(nrReceipts, 150*time.Second), + // receiptCache: newFifoCache(nrReceipts, 150*time.Second), receiptCache: newFifoCache(nrReceipts, gocache.NoExpiration), attestedEnclavesCache: newLFUCache[[]byte, *AttestedEnclave](logger, nrEnclaves), From 9d0b82e5f9c8d31d69477f0cfb042290590eb7cc Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Fri, 27 Dec 2024 15:40:35 +0200 Subject: [PATCH 07/11] fixes --- go/enclave/rpc/GetTransaction.go | 30 +++++++++++++++++-- go/enclave/rpc/GetTransactionReceipt.go | 29 +++++++++++++------ go/enclave/storage/cache_service.go | 38 +++++++++++++++++-------- 3 files changed, 75 insertions(+), 22 deletions(-) diff --git a/go/enclave/rpc/GetTransaction.go b/go/enclave/rpc/GetTransaction.go index 42861572c..efe1d1481 100644 --- a/go/enclave/rpc/GetTransaction.go +++ b/go/enclave/rpc/GetTransaction.go @@ -5,6 +5,10 @@ import ( "fmt" "math/big" + "github.com/ten-protocol/go-ten/go/enclave/storage" + + "github.com/ten-protocol/go-ten/go/common/log" + "github.com/ten-protocol/go-ten/go/enclave/core" gethcommon "github.com/ethereum/go-ethereum/common" @@ -34,11 +38,34 @@ func GetTransactionValidate(reqParams []any, builder *CallBuilder[gethcommon.Has } func GetTransactionExecute(builder *CallBuilder[gethcommon.Hash, RpcTransaction], rpc *EncryptionManager) error { + txHash := *builder.Param + requester := builder.VK.AccountAddress + + // first try the cache for recent transactions + rec, err := rpc.cacheService.ReadReceipt(builder.ctx, txHash) + // there is an explicit entry in the cache that the tx was not found + if err != nil && errors.Is(err, storage.ReceiptDoesNotExist) { + builder.Status = NotFound + return nil + } + + if rec != nil { + rpc.logger.Info("Cache hit for receipt", log.TxKey, txHash) + // authorise - only the signer can request the transaction + if rec.From.Hex() != requester.Hex() { + builder.Status = NotAuthorised + return nil + } + builder.ReturnValue = newRPCTransaction(rec.Tx, rec.Receipt.BlockHash, rec.Receipt.BlockNumber.Uint64(), uint64(rec.Receipt.TransactionIndex), rpc.config.BaseFee, *rec.From) + return nil + } + // Unlike in the Geth impl, we do not try and retrieve unconfirmed transactions from the mempool. tx, blockHash, blockNumber, index, err := rpc.storage.GetTransaction(builder.ctx, *builder.Param) if err != nil { if errors.Is(err, errutil.ErrNotFound) { builder.Status = NotFound + rpc.cacheService.ReceiptDoesNotExist(txHash) return nil } return err @@ -50,9 +77,8 @@ func GetTransactionExecute(builder *CallBuilder[gethcommon.Hash, RpcTransaction] } // authorise - only the signer can request the transaction - if sender.Hex() != builder.VK.AccountAddress.Hex() { + if sender.Hex() != requester.Hex() { builder.Status = NotAuthorised - // builder.ReturnValue= []byte{} return nil } diff --git a/go/enclave/rpc/GetTransactionReceipt.go b/go/enclave/rpc/GetTransactionReceipt.go index 103922f9a..7027f7f02 100644 --- a/go/enclave/rpc/GetTransactionReceipt.go +++ b/go/enclave/rpc/GetTransactionReceipt.go @@ -43,7 +43,13 @@ func GetTransactionReceiptExecute(builder *CallBuilder[gethcommon.Hash, map[stri // first try the cache for recent transactions result, err := fetchFromCache(builder.ctx, rpc.storage, rpc.cacheService, txHash, requester) - if err != nil { + // there is an explicit entry in the cache that the tx was not found + if err != nil && errors.Is(err, storage.ReceiptDoesNotExist) { + builder.Status = NotFound + return nil + } + // unexpected error + if err != nil && !errors.Is(err, errutil.ErrNotFound) { return err } @@ -59,18 +65,25 @@ func GetTransactionReceiptExecute(builder *CallBuilder[gethcommon.Hash, map[stri return fmt.Errorf("could not retrieve transaction receipt in eth_getTransactionReceipt request. Cause: %w", err) } if !exists { + rpc.cacheService.ReceiptDoesNotExist(txHash) builder.Status = NotFound return nil } - // try the cache again in case the tx was commited in the meantime + // try the cache again in case the tx was committed in the meantime result, err = fetchFromCache(builder.ctx, rpc.storage, rpc.cacheService, txHash, requester) - if err != nil { + // there is an explicit entry in the cache that the tx was not found + if err != nil && errors.Is(err, storage.ReceiptDoesNotExist) { + builder.Status = NotFound + return nil + } + // unexpected error + if err != nil && !errors.Is(err, errutil.ErrNotFound) { return err } if result != nil { - rpc.logger.Info("Cache hit for receipt", log.TxKey, txHash) + rpc.logger.Info("Cache hit for receipt after", log.TxKey, txHash) builder.ReturnValue = &result return nil } @@ -94,9 +107,9 @@ func GetTransactionReceiptExecute(builder *CallBuilder[gethcommon.Hash, map[stri } func fetchFromCache(ctx context.Context, storage storage.Storage, cacheService *storage.CacheService, txHash gethcommon.Hash, requester *gethcommon.Address) (map[string]interface{}, error) { - rec, _ := cacheService.ReadReceipt(ctx, txHash) - if rec == nil { - return nil, nil + rec, err := cacheService.ReadReceipt(ctx, txHash) + if err != nil { + return nil, err } // receipt found in cache @@ -129,7 +142,7 @@ func fetchFromCache(ctx context.Context, storage storage.Storage, cacheService * r := marshalReceipt(rec.Receipt, logs, rec.From, rec.To) // after the receipt was requested by a user remove it from the cache - err := cacheService.DelReceipt(ctx, txHash) + err = cacheService.DelReceipt(ctx, txHash) if err != nil { return nil, err } diff --git a/go/enclave/storage/cache_service.go b/go/enclave/storage/cache_service.go index b49d096a1..86b03e36e 100644 --- a/go/enclave/storage/cache_service.go +++ b/go/enclave/storage/cache_service.go @@ -3,6 +3,7 @@ package storage import ( "context" "encoding/binary" + "errors" "fmt" "math/big" "time" @@ -153,18 +154,6 @@ func (cs *CacheService) CacheBlock(ctx context.Context, b *types.Header) { cacheValue(ctx, cs.blockCache, cs.logger, b.Hash().Bytes(), b) } -func (cs *CacheService) CacheReceipts(results core.TxExecResults) { - receipts := make(map[string]any) - for _, txExecResult := range results { - receipts[txExecResult.TxWithSender.Tx.Hash().String()] = &CachedReceipt{ - Receipt: txExecResult.Receipt, - From: txExecResult.TxWithSender.Sender, - To: txExecResult.TxWithSender.Tx.To(), - } - } - cs.receiptCache.SetAll(receipts) -} - func (cs *CacheService) CacheBatch(ctx context.Context, batch *core.Batch) { cacheValue(ctx, cs.batchCacheBySeqNo, cs.logger, batch.SeqNo().Uint64(), batch.Header) cacheValue(ctx, cs.seqCacheByHash, cs.logger, batch.Hash().Bytes(), batch.SeqNo()) @@ -228,12 +217,34 @@ func (cs *CacheService) ReadEventTopic(ctx context.Context, topic []byte, eventT return getCachedValue(ctx, cs.eventTopicCache, cs.logger, key, onCacheMiss, true) } +// CachedReceipt - when all values are nil, it means there is no receipt type CachedReceipt struct { Receipt *types.Receipt + Tx *types.Transaction From *gethcommon.Address To *gethcommon.Address } +var ReceiptDoesNotExist = errors.New("receipt does not exist") + +func (cs *CacheService) CacheReceipts(results core.TxExecResults) { + receipts := make(map[string]any) + for _, txExecResult := range results { + receipts[txExecResult.TxWithSender.Tx.Hash().String()] = &CachedReceipt{ + Receipt: txExecResult.Receipt, + Tx: txExecResult.TxWithSender.Tx, + From: txExecResult.TxWithSender.Sender, + To: txExecResult.TxWithSender.Tx.To(), + } + cs.logger.Info("Cache receipt", "tx", txExecResult.TxWithSender.Tx.Hash().String()) + } + cs.receiptCache.SetAll(receipts) +} + +func (cs *CacheService) ReceiptDoesNotExist(txHash gethcommon.Hash) { + cs.receiptCache.Set(txHash.String(), &CachedReceipt{}) +} + func (cs *CacheService) ReadReceipt(_ context.Context, txHash gethcommon.Hash) (*CachedReceipt, error) { r, found := cs.receiptCache.Get(txHash.String()) if !found { @@ -243,6 +254,9 @@ func (cs *CacheService) ReadReceipt(_ context.Context, txHash gethcommon.Hash) ( if !ok { return nil, fmt.Errorf("should not happen. invalid cached receipt") } + if cr.Receipt == nil { + return nil, ReceiptDoesNotExist + } return cr, nil } From eb8b015699462e885974e23a9ffb1fb2c0b6b833 Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Fri, 27 Dec 2024 16:20:49 +0200 Subject: [PATCH 08/11] fixes --- go/enclave/rpc/GetTransactionReceipt.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/go/enclave/rpc/GetTransactionReceipt.go b/go/enclave/rpc/GetTransactionReceipt.go index 7027f7f02..a918427d1 100644 --- a/go/enclave/rpc/GetTransactionReceipt.go +++ b/go/enclave/rpc/GetTransactionReceipt.go @@ -36,6 +36,8 @@ func GetTransactionReceiptValidate(reqParams []any, builder *CallBuilder[gethcom return nil } +var NotAuthorisedErr = errors.New("not authorised") + func GetTransactionReceiptExecute(builder *CallBuilder[gethcommon.Hash, map[string]interface{}], rpc *EncryptionManager) error { txHash := *builder.Param requester := builder.VK.AccountAddress @@ -48,6 +50,10 @@ func GetTransactionReceiptExecute(builder *CallBuilder[gethcommon.Hash, map[stri builder.Status = NotFound return nil } + if err != nil && errors.Is(err, NotAuthorisedErr) { + builder.Status = NotAuthorised + return nil + } // unexpected error if err != nil && !errors.Is(err, errutil.ErrNotFound) { return err @@ -81,6 +87,10 @@ func GetTransactionReceiptExecute(builder *CallBuilder[gethcommon.Hash, map[stri if err != nil && !errors.Is(err, errutil.ErrNotFound) { return err } + if err != nil && errors.Is(err, NotAuthorisedErr) { + builder.Status = NotAuthorised + return nil + } if result != nil { rpc.logger.Info("Cache hit for receipt after", log.TxKey, txHash) @@ -116,7 +126,7 @@ func fetchFromCache(ctx context.Context, storage storage.Storage, cacheService * // for simplicity only the tx sender will access the cache // check whether the requester is the sender if *rec.From != *requester { - return nil, nil + return nil, NotAuthorisedErr } logs := rec.Receipt.Logs From d928e35b4764635aa147f5522ef3c0068388c2bb Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Fri, 27 Dec 2024 16:44:28 +0200 Subject: [PATCH 09/11] fixes --- go/enclave/rpc/GetTransaction.go | 4 +++- go/enclave/rpc/GetTransactionReceipt.go | 30 ++++--------------------- go/enclave/storage/cache_service.go | 3 +-- 3 files changed, 8 insertions(+), 29 deletions(-) diff --git a/go/enclave/rpc/GetTransaction.go b/go/enclave/rpc/GetTransaction.go index efe1d1481..fed8375b5 100644 --- a/go/enclave/rpc/GetTransaction.go +++ b/go/enclave/rpc/GetTransaction.go @@ -50,7 +50,7 @@ func GetTransactionExecute(builder *CallBuilder[gethcommon.Hash, RpcTransaction] } if rec != nil { - rpc.logger.Info("Cache hit for receipt", log.TxKey, txHash) + rpc.logger.Info("Cache hit for tx", log.TxKey, txHash) // authorise - only the signer can request the transaction if rec.From.Hex() != requester.Hex() { builder.Status = NotAuthorised @@ -60,6 +60,8 @@ func GetTransactionExecute(builder *CallBuilder[gethcommon.Hash, RpcTransaction] return nil } + rpc.logger.Info("Cache miss for tx", log.TxKey, txHash) + // Unlike in the Geth impl, we do not try and retrieve unconfirmed transactions from the mempool. tx, blockHash, blockNumber, index, err := rpc.storage.GetTransaction(builder.ctx, *builder.Param) if err != nil { diff --git a/go/enclave/rpc/GetTransactionReceipt.go b/go/enclave/rpc/GetTransactionReceipt.go index a918427d1..7562537d2 100644 --- a/go/enclave/rpc/GetTransactionReceipt.go +++ b/go/enclave/rpc/GetTransactionReceipt.go @@ -76,28 +76,6 @@ func GetTransactionReceiptExecute(builder *CallBuilder[gethcommon.Hash, map[stri return nil } - // try the cache again in case the tx was committed in the meantime - result, err = fetchFromCache(builder.ctx, rpc.storage, rpc.cacheService, txHash, requester) - // there is an explicit entry in the cache that the tx was not found - if err != nil && errors.Is(err, storage.ReceiptDoesNotExist) { - builder.Status = NotFound - return nil - } - // unexpected error - if err != nil && !errors.Is(err, errutil.ErrNotFound) { - return err - } - if err != nil && errors.Is(err, NotAuthorisedErr) { - builder.Status = NotAuthorised - return nil - } - - if result != nil { - rpc.logger.Info("Cache hit for receipt after", log.TxKey, txHash) - builder.ReturnValue = &result - return nil - } - // We retrieve the transaction receipt. receipt, err := rpc.storage.GetFilteredInternalReceipt(builder.ctx, txHash, requester, false) if err != nil { @@ -152,10 +130,10 @@ func fetchFromCache(ctx context.Context, storage storage.Storage, cacheService * r := marshalReceipt(rec.Receipt, logs, rec.From, rec.To) // after the receipt was requested by a user remove it from the cache - err = cacheService.DelReceipt(ctx, txHash) - if err != nil { - return nil, err - } + //err = cacheService.DelReceipt(ctx, txHash) + //if err != nil { + // return nil, err + //} return r, nil } diff --git a/go/enclave/storage/cache_service.go b/go/enclave/storage/cache_service.go index 86b03e36e..f6fda973c 100644 --- a/go/enclave/storage/cache_service.go +++ b/go/enclave/storage/cache_service.go @@ -106,8 +106,7 @@ func NewCacheService(logger gethlog.Logger, testMode bool) *CacheService { eventTypeCache: newLFUCache[[]byte, *enclavedb.EventType](logger, nrEventTypes), eventTopicCache: newLFUCache[[]byte, *enclavedb.EventTopic](logger, nrEventTypes), - // receiptCache: newFifoCache(nrReceipts, 150*time.Second), - receiptCache: newFifoCache(nrReceipts, gocache.NoExpiration), + receiptCache: newFifoCache(nrReceipts, 180*time.Second), attestedEnclavesCache: newLFUCache[[]byte, *AttestedEnclave](logger, nrEnclaves), // cache the latest received batches to avoid a lookup when streaming it back to the host after processing From 9eef2d9cb5116dfe9c85bc1c757d394e1c0f0487 Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Fri, 27 Dec 2024 17:09:05 +0200 Subject: [PATCH 10/11] fixes --- go/enclave/rpc/GetTransaction.go | 4 ++-- go/enclave/rpc/GetTransactionReceipt.go | 13 ++++--------- go/enclave/storage/cache_service.go | 5 +++-- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/go/enclave/rpc/GetTransaction.go b/go/enclave/rpc/GetTransaction.go index fed8375b5..0e5631f42 100644 --- a/go/enclave/rpc/GetTransaction.go +++ b/go/enclave/rpc/GetTransaction.go @@ -50,7 +50,7 @@ func GetTransactionExecute(builder *CallBuilder[gethcommon.Hash, RpcTransaction] } if rec != nil { - rpc.logger.Info("Cache hit for tx", log.TxKey, txHash) + rpc.logger.Debug("Cache hit for tx", log.TxKey, txHash) // authorise - only the signer can request the transaction if rec.From.Hex() != requester.Hex() { builder.Status = NotAuthorised @@ -60,7 +60,7 @@ func GetTransactionExecute(builder *CallBuilder[gethcommon.Hash, RpcTransaction] return nil } - rpc.logger.Info("Cache miss for tx", log.TxKey, txHash) + rpc.logger.Debug("Cache miss for tx", log.TxKey, txHash) // Unlike in the Geth impl, we do not try and retrieve unconfirmed transactions from the mempool. tx, blockHash, blockNumber, index, err := rpc.storage.GetTransaction(builder.ctx, *builder.Param) diff --git a/go/enclave/rpc/GetTransactionReceipt.go b/go/enclave/rpc/GetTransactionReceipt.go index 7562537d2..049bf1180 100644 --- a/go/enclave/rpc/GetTransactionReceipt.go +++ b/go/enclave/rpc/GetTransactionReceipt.go @@ -60,12 +60,12 @@ func GetTransactionReceiptExecute(builder *CallBuilder[gethcommon.Hash, map[stri } if result != nil { - rpc.logger.Info("Cache hit for receipt", log.TxKey, txHash) + rpc.logger.Debug("Cache hit for receipt", log.TxKey, txHash) builder.ReturnValue = &result return nil } - rpc.logger.Info("Cache miss for receipt", log.TxKey, txHash.String()) + rpc.logger.Debug("Cache miss for receipt", log.TxKey, txHash.String()) exists, err := rpc.storage.ExistsTransactionReceipt(builder.ctx, txHash) if err != nil { return fmt.Errorf("could not retrieve transaction receipt in eth_getTransactionReceipt request. Cause: %w", err) @@ -79,7 +79,7 @@ func GetTransactionReceiptExecute(builder *CallBuilder[gethcommon.Hash, map[stri // We retrieve the transaction receipt. receipt, err := rpc.storage.GetFilteredInternalReceipt(builder.ctx, txHash, requester, false) if err != nil { - rpc.logger.Trace("error getting tx receipt", log.TxKey, txHash, log.ErrKey, err) + rpc.logger.Trace("Error getting tx receipt", log.TxKey, txHash, log.ErrKey, err) if errors.Is(err, errutil.ErrNotFound) { builder.Status = NotAuthorised return nil @@ -127,13 +127,8 @@ func fetchFromCache(ctx context.Context, storage storage.Storage, cacheService * } } } - r := marshalReceipt(rec.Receipt, logs, rec.From, rec.To) - // after the receipt was requested by a user remove it from the cache - //err = cacheService.DelReceipt(ctx, txHash) - //if err != nil { - // return nil, err - //} + r := marshalReceipt(rec.Receipt, logs, rec.From, rec.To) return r, nil } diff --git a/go/enclave/storage/cache_service.go b/go/enclave/storage/cache_service.go index f6fda973c..0d86a77f5 100644 --- a/go/enclave/storage/cache_service.go +++ b/go/enclave/storage/cache_service.go @@ -87,6 +87,7 @@ func NewCacheService(logger gethlog.Logger, testMode bool) *CacheService { nrEnclaves := 20 nrReceipts := 15_000 // ~100M + receiptsTimeout := 4 * time.Minute if testMode { nrReceipts = 2500 } @@ -106,7 +107,7 @@ func NewCacheService(logger gethlog.Logger, testMode bool) *CacheService { eventTypeCache: newLFUCache[[]byte, *enclavedb.EventType](logger, nrEventTypes), eventTopicCache: newLFUCache[[]byte, *enclavedb.EventTopic](logger, nrEventTypes), - receiptCache: newFifoCache(nrReceipts, 180*time.Second), + receiptCache: newFifoCache(nrReceipts, receiptsTimeout), attestedEnclavesCache: newLFUCache[[]byte, *AttestedEnclave](logger, nrEnclaves), // cache the latest received batches to avoid a lookup when streaming it back to the host after processing @@ -235,7 +236,7 @@ func (cs *CacheService) CacheReceipts(results core.TxExecResults) { From: txExecResult.TxWithSender.Sender, To: txExecResult.TxWithSender.Tx.To(), } - cs.logger.Info("Cache receipt", "tx", txExecResult.TxWithSender.Tx.Hash().String()) + cs.logger.Debug("Cache receipt", "tx", txExecResult.TxWithSender.Tx.Hash().String()) } cs.receiptCache.SetAll(receipts) } From 56d8ddebfd67cf7840e7e8412219609a8c3673e5 Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Sun, 29 Dec 2024 13:48:33 +0200 Subject: [PATCH 11/11] fixes --- go/enclave/rpc/GetTransaction.go | 23 ++++++++--------------- go/enclave/rpc/GetTransactionReceipt.go | 12 +++++------- go/enclave/rpc/vk_utils.go | 1 - go/enclave/storage/enclavedb/batch.go | 21 ++++++++++++++------- go/enclave/storage/interfaces.go | 2 +- go/enclave/storage/storage.go | 2 +- 6 files changed, 29 insertions(+), 32 deletions(-) diff --git a/go/enclave/rpc/GetTransaction.go b/go/enclave/rpc/GetTransaction.go index 0e5631f42..767c6d525 100644 --- a/go/enclave/rpc/GetTransaction.go +++ b/go/enclave/rpc/GetTransaction.go @@ -9,8 +9,6 @@ import ( "github.com/ten-protocol/go-ten/go/common/log" - "github.com/ten-protocol/go-ten/go/enclave/core" - gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" @@ -52,7 +50,7 @@ func GetTransactionExecute(builder *CallBuilder[gethcommon.Hash, RpcTransaction] if rec != nil { rpc.logger.Debug("Cache hit for tx", log.TxKey, txHash) // authorise - only the signer can request the transaction - if rec.From.Hex() != requester.Hex() { + if *rec.From != *requester { builder.Status = NotAuthorised return nil } @@ -63,23 +61,18 @@ func GetTransactionExecute(builder *CallBuilder[gethcommon.Hash, RpcTransaction] rpc.logger.Debug("Cache miss for tx", log.TxKey, txHash) // Unlike in the Geth impl, we do not try and retrieve unconfirmed transactions from the mempool. - tx, blockHash, blockNumber, index, err := rpc.storage.GetTransaction(builder.ctx, *builder.Param) - if err != nil { - if errors.Is(err, errutil.ErrNotFound) { - builder.Status = NotFound - rpc.cacheService.ReceiptDoesNotExist(txHash) - return nil - } - return err + tx, blockHash, blockNumber, index, sender, err := rpc.storage.GetTransaction(builder.ctx, *builder.Param) + if err != nil && errors.Is(err, errutil.ErrNotFound) { + builder.Status = NotFound + rpc.cacheService.ReceiptDoesNotExist(txHash) + return nil } - - sender, err := core.GetExternalTxSigner(tx) if err != nil { - return fmt.Errorf("could not recover the tx %s sender. Cause: %w", tx.Hash(), err) + return err } // authorise - only the signer can request the transaction - if sender.Hex() != requester.Hex() { + if sender != *requester { builder.Status = NotAuthorised return nil } diff --git a/go/enclave/rpc/GetTransactionReceipt.go b/go/enclave/rpc/GetTransactionReceipt.go index 049bf1180..7bb90eb0b 100644 --- a/go/enclave/rpc/GetTransactionReceipt.go +++ b/go/enclave/rpc/GetTransactionReceipt.go @@ -100,13 +100,6 @@ func fetchFromCache(ctx context.Context, storage storage.Storage, cacheService * return nil, err } - // receipt found in cache - // for simplicity only the tx sender will access the cache - // check whether the requester is the sender - if *rec.From != *requester { - return nil, NotAuthorisedErr - } - logs := rec.Receipt.Logs // filter out the logs that the sender can't read // doesn't apply to contract creation (when to=nil) @@ -128,6 +121,11 @@ func fetchFromCache(ctx context.Context, storage storage.Storage, cacheService * } } + // check whether the requester is the sender or the requester can see any logs + if (*rec.From != *requester) && (len(logs) == 0) { + return nil, NotAuthorisedErr + } + r := marshalReceipt(rec.Receipt, logs, rec.From, rec.To) return r, nil } diff --git a/go/enclave/rpc/vk_utils.go b/go/enclave/rpc/vk_utils.go index 8b60f6e46..7b1e04021 100644 --- a/go/enclave/rpc/vk_utils.go +++ b/go/enclave/rpc/vk_utils.go @@ -147,7 +147,6 @@ func withVKEncryption[P any, R any]( } if builder.Status == NotAuthorised { // if the requested resource was not found, return an empty response - // todo - this must be encrypted - but we have some logic that expects it unencrypted, which is a bug return responses.AsEncryptedError(errors.New("not authorised"), vk), nil } diff --git a/go/enclave/storage/enclavedb/batch.go b/go/enclave/storage/enclavedb/batch.go index d215c8058..c2a40b9fe 100644 --- a/go/enclave/storage/enclavedb/batch.go +++ b/go/enclave/storage/enclavedb/batch.go @@ -281,9 +281,14 @@ func ExistsReceipt(ctx context.Context, db *sql.DB, txHash common.L2TxHash) (boo return cnt > 0, nil } -func ReadTransaction(ctx context.Context, db *sql.DB, txHash gethcommon.Hash) (*types.Transaction, common.L2BatchHash, uint64, uint64, error) { +func ReadTransaction(ctx context.Context, db *sql.DB, txHash gethcommon.Hash) (*types.Transaction, common.L2BatchHash, uint64, uint64, gethcommon.Address, error) { row := db.QueryRowContext(ctx, - "select tx.content, batch.hash, batch.height, tx.idx from receipt join tx on tx.id=receipt.tx join batch on batch.sequence=receipt.batch where batch.is_canonical=true and tx.hash=?", + "select tx.content, batch.hash, batch.height, tx.idx, eoa.address "+ + "from receipt "+ + "join tx on tx.id=receipt.tx "+ + "join batch on batch.sequence=receipt.batch "+ + "join externally_owned_account eoa on eoa.id = tx.sender_address "+ + "where batch.is_canonical=true and tx.hash=?", txHash.Bytes()) // tx, batch, height, idx @@ -291,21 +296,23 @@ func ReadTransaction(ctx context.Context, db *sql.DB, txHash gethcommon.Hash) (* var batchHash []byte var height uint64 var idx uint64 - err := row.Scan(&txData, &batchHash, &height, &idx) + var sender []byte + err := row.Scan(&txData, &batchHash, &height, &idx, &sender) if err != nil { if errors.Is(err, sql.ErrNoRows) { // make sure the error is converted to obscuro-wide not found error - return nil, gethcommon.Hash{}, 0, 0, errutil.ErrNotFound + return nil, gethcommon.Hash{}, 0, 0, gethcommon.Address{}, errutil.ErrNotFound } - return nil, gethcommon.Hash{}, 0, 0, err + return nil, gethcommon.Hash{}, 0, 0, gethcommon.Address{}, err } tx := new(common.L2Tx) if err := rlp.DecodeBytes(txData, tx); err != nil { - return nil, gethcommon.Hash{}, 0, 0, fmt.Errorf("could not decode L2 transaction. Cause: %w", err) + return nil, gethcommon.Hash{}, 0, 0, gethcommon.Address{}, fmt.Errorf("could not decode L2 transaction. Cause: %w", err) } batch := gethcommon.Hash{} batch.SetBytes(batchHash) - return tx, batch, height, idx, nil + senderAddress := gethcommon.BytesToAddress(sender) + return tx, batch, height, idx, senderAddress, nil } func ReadBatchTransactions(ctx context.Context, db *sql.DB, height uint64) ([]*common.L2Tx, error) { diff --git a/go/enclave/storage/interfaces.go b/go/enclave/storage/interfaces.go index 6f44770dc..9b641de17 100644 --- a/go/enclave/storage/interfaces.go +++ b/go/enclave/storage/interfaces.go @@ -96,7 +96,7 @@ type SharedSecretStorage interface { type TransactionStorage interface { // GetTransaction - returns the positional metadata of the tx by hash - GetTransaction(ctx context.Context, txHash common.L2TxHash) (*types.Transaction, common.L2BatchHash, uint64, uint64, error) + GetTransaction(ctx context.Context, txHash common.L2TxHash) (*types.Transaction, common.L2BatchHash, uint64, uint64, gethcommon.Address, error) // GetFilteredInternalReceipt - returns the receipt of a tx with event logs visible to the requester GetFilteredInternalReceipt(ctx context.Context, txHash common.L2TxHash, requester *gethcommon.Address, syntheticTx bool) (*core.InternalReceipt, error) ExistsTransactionReceipt(ctx context.Context, txHash common.L2TxHash) (bool, error) diff --git a/go/enclave/storage/storage.go b/go/enclave/storage/storage.go index 04728f7ce..fbefa4730 100644 --- a/go/enclave/storage/storage.go +++ b/go/enclave/storage/storage.go @@ -434,7 +434,7 @@ func (s *storageImpl) EmptyStateDB() (*state.StateDB, error) { return statedb, nil } -func (s *storageImpl) GetTransaction(ctx context.Context, txHash gethcommon.Hash) (*types.Transaction, common.L2BatchHash, uint64, uint64, error) { +func (s *storageImpl) GetTransaction(ctx context.Context, txHash common.L2TxHash) (*types.Transaction, common.L2BatchHash, uint64, uint64, gethcommon.Address, error) { defer s.logDuration("GetTransaction", measure.NewStopwatch()) return enclavedb.ReadTransaction(ctx, s.db.GetSQLDB(), txHash) }