From 1158ad1b043c29adf7341ecb52fceaec544dc4d4 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Mon, 9 Sep 2024 19:51:09 +0800 Subject: [PATCH 1/8] polygon/heimdall: Split read functions into reader (#11924) - Create `heimdall.Reader` for future use in `bor_*` API - Make `AssembleReader` and `NewReader` leaner by not requiring full `BorConfig` --- eth/backend.go | 5 +- eth/stagedsync/stage_polygon_sync.go | 3 +- polygon/heimdall/reader.go | 57 +++++++++++++++++++ polygon/heimdall/service.go | 26 ++++----- polygon/heimdall/service_test.go | 3 +- .../heimdall/span_block_producers_tracker.go | 35 ++++++------ 6 files changed, 95 insertions(+), 34 deletions(-) create mode 100644 polygon/heimdall/reader.go diff --git a/eth/backend.go b/eth/backend.go index 37d968df51c..5b7b6c91d73 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -550,8 +550,9 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger } if config.PolygonSync { - polygonBridge = bridge.Assemble(config.Dirs.DataDir, logger, consensusConfig.(*borcfg.BorConfig), heimdallClient) - heimdallService = heimdall.AssembleService(consensusConfig.(*borcfg.BorConfig), config.HeimdallURL, dirs.DataDir, tmpdir, logger) + borConfig := consensusConfig.(*borcfg.BorConfig) + polygonBridge = bridge.Assemble(config.Dirs.DataDir, logger, borConfig, heimdallClient) + heimdallService = heimdall.AssembleService(borConfig.CalculateSprintNumber, config.HeimdallURL, dirs.DataDir, tmpdir, logger) backend.polygonBridge = polygonBridge } diff --git a/eth/stagedsync/stage_polygon_sync.go b/eth/stagedsync/stage_polygon_sync.go index a5818b7994a..6ddd4eba995 100644 --- a/eth/stagedsync/stage_polygon_sync.go +++ b/eth/stagedsync/stage_polygon_sync.go @@ -97,7 +97,8 @@ func NewPolygonSyncStageCfg( txActionStream: txActionStream, } borConfig := chainConfig.Bor.(*borcfg.BorConfig) - heimdallService := heimdall.NewService(borConfig, heimdallClient, heimdallStore, logger) + heimdallReader := heimdall.NewReader(borConfig.CalculateSprintNumber, heimdallStore, logger) + heimdallService := heimdall.NewService(borConfig.CalculateSprintNumber, heimdallClient, heimdallStore, logger, heimdallReader) bridgeService := bridge.NewBridge(bridgeStore, logger, borConfig, heimdallClient, nil) p2pService := p2p.NewService(maxPeers, logger, sentry, statusDataProvider.GetStatusData) checkpointVerifier := polygonsync.VerifyCheckpointHeaders diff --git a/polygon/heimdall/reader.go b/polygon/heimdall/reader.go new file mode 100644 index 00000000000..7ac14bb9f64 --- /dev/null +++ b/polygon/heimdall/reader.go @@ -0,0 +1,57 @@ +package heimdall + +import ( + "context" + + libcommon "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/log/v3" + "github.com/erigontech/erigon/polygon/bor/valset" +) + +type Reader struct { + logger log.Logger + store ServiceStore + spanBlockProducersTracker *spanBlockProducersTracker +} + +// AssembleReader creates and opens the MDBX store. For use cases where the store is only being read from. Must call Close. +func AssembleReader(ctx context.Context, calculateSprintNumber CalculateSprintNumberFunc, dataDir string, tmpDir string, logger log.Logger) (*Reader, error) { + store := NewMdbxServiceStore(logger, dataDir, tmpDir) + + err := store.Prepare(ctx) + if err != nil { + return nil, err + } + + return NewReader(calculateSprintNumber, store, logger), nil +} + +func NewReader(calculateSprintNumber CalculateSprintNumberFunc, store ServiceStore, logger log.Logger) *Reader { + return &Reader{ + logger: logger, + store: store, + spanBlockProducersTracker: newSpanBlockProducersTracker(logger, calculateSprintNumber, store.SpanBlockProducerSelections()), + } +} + +func (r *Reader) Span(ctx context.Context, id uint64) (*Span, bool, error) { + return r.store.Spans().Entity(ctx, id) +} + +func (r *Reader) CheckpointsFromBlock(ctx context.Context, startBlock uint64) (Waypoints, error) { + entities, err := r.store.Checkpoints().RangeFromBlockNum(ctx, startBlock) + return libcommon.SliceMap(entities, castEntityToWaypoint[*Checkpoint]), err +} + +func (r *Reader) MilestonesFromBlock(ctx context.Context, startBlock uint64) (Waypoints, error) { + entities, err := r.store.Milestones().RangeFromBlockNum(ctx, startBlock) + return libcommon.SliceMap(entities, castEntityToWaypoint[*Milestone]), err +} + +func (r *Reader) Producers(ctx context.Context, blockNum uint64) (*valset.ValidatorSet, error) { + return r.spanBlockProducersTracker.Producers(ctx, blockNum) +} + +func (r *Reader) Close() { + r.store.Close() +} diff --git a/polygon/heimdall/service.go b/polygon/heimdall/service.go index 3f7715698ab..2786024601a 100644 --- a/polygon/heimdall/service.go +++ b/polygon/heimdall/service.go @@ -26,7 +26,6 @@ import ( libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/log/v3" - "github.com/erigontech/erigon/polygon/bor/borcfg" "github.com/erigontech/erigon/polygon/bor/valset" "github.com/erigontech/erigon/polygon/polygoncommon" ) @@ -46,23 +45,25 @@ type Service interface { type service struct { logger log.Logger store ServiceStore + reader *Reader checkpointScraper *scraper[*Checkpoint] milestoneScraper *scraper[*Milestone] spanScraper *scraper[*Span] spanBlockProducersTracker *spanBlockProducersTracker } -func AssembleService(borConfig *borcfg.BorConfig, heimdallUrl string, dataDir string, tmpDir string, logger log.Logger) Service { +func AssembleService(calculateSprintNumberFn CalculateSprintNumberFunc, heimdallUrl string, dataDir string, tmpDir string, logger log.Logger) Service { store := NewMdbxServiceStore(logger, dataDir, tmpDir) client := NewHeimdallClient(heimdallUrl, logger) - return NewService(borConfig, client, store, logger) + reader := NewReader(calculateSprintNumberFn, store, logger) + return NewService(calculateSprintNumberFn, client, store, logger, reader) } -func NewService(borConfig *borcfg.BorConfig, client HeimdallClient, store ServiceStore, logger log.Logger) Service { - return newService(borConfig, client, store, logger) +func NewService(calculateSprintNumberFn CalculateSprintNumberFunc, client HeimdallClient, store ServiceStore, logger log.Logger, reader *Reader) Service { + return newService(calculateSprintNumberFn, client, store, logger, reader) } -func newService(borConfig *borcfg.BorConfig, client HeimdallClient, store ServiceStore, logger log.Logger) *service { +func newService(calculateSprintNumberFn CalculateSprintNumberFunc, client HeimdallClient, store ServiceStore, logger log.Logger, reader *Reader) *service { checkpointFetcher := newCheckpointFetcher(client, logger) milestoneFetcher := newMilestoneFetcher(client, logger) spanFetcher := newSpanFetcher(client, logger) @@ -101,10 +102,11 @@ func newService(borConfig *borcfg.BorConfig, client HeimdallClient, store Servic return &service{ logger: logger, store: store, + reader: reader, checkpointScraper: checkpointScraper, milestoneScraper: milestoneScraper, spanScraper: spanScraper, - spanBlockProducersTracker: newSpanBlockProducersTracker(logger, borConfig, store.SpanBlockProducerSelections()), + spanBlockProducersTracker: newSpanBlockProducersTracker(logger, calculateSprintNumberFn, store.SpanBlockProducerSelections()), } } @@ -164,7 +166,7 @@ func newSpanFetcher(client HeimdallClient, logger log.Logger) entityFetcher[*Spa } func (s *service) Span(ctx context.Context, id uint64) (*Span, bool, error) { - return s.store.Spans().Entity(ctx, id) + return s.reader.Span(ctx, id) } func castEntityToWaypoint[TEntity Waypoint](entity TEntity) Waypoint { @@ -220,17 +222,15 @@ func (s *service) synchronizeSpans(ctx context.Context) error { } func (s *service) CheckpointsFromBlock(ctx context.Context, startBlock uint64) (Waypoints, error) { - entities, err := s.store.Checkpoints().RangeFromBlockNum(ctx, startBlock) - return libcommon.SliceMap(entities, castEntityToWaypoint[*Checkpoint]), err + return s.reader.CheckpointsFromBlock(ctx, startBlock) } func (s *service) MilestonesFromBlock(ctx context.Context, startBlock uint64) (Waypoints, error) { - entities, err := s.store.Milestones().RangeFromBlockNum(ctx, startBlock) - return libcommon.SliceMap(entities, castEntityToWaypoint[*Milestone]), err + return s.reader.MilestonesFromBlock(ctx, startBlock) } func (s *service) Producers(ctx context.Context, blockNum uint64) (*valset.ValidatorSet, error) { - return s.spanBlockProducersTracker.Producers(ctx, blockNum) + return s.reader.Producers(ctx, blockNum) } func (s *service) RegisterMilestoneObserver(callback func(*Milestone), opts ...ObserverOption) polygoncommon.UnregisterFunc { diff --git a/polygon/heimdall/service_test.go b/polygon/heimdall/service_test.go index 7768cfbae70..3e176af8eec 100644 --- a/polygon/heimdall/service_test.go +++ b/polygon/heimdall/service_test.go @@ -77,7 +77,8 @@ func (suite *ServiceTestSuite) SetupSuite() { suite.setupSpans() suite.setupCheckpoints() suite.setupMilestones() - suite.service = newService(borConfig, suite.client, store, logger) + reader := NewReader(borConfig.CalculateSprintNumber, store, logger) + suite.service = newService(borConfig.CalculateSprintNumber, suite.client, store, logger, reader) err := suite.service.store.Prepare(suite.ctx) require.NoError(suite.T(), err) diff --git a/polygon/heimdall/span_block_producers_tracker.go b/polygon/heimdall/span_block_producers_tracker.go index a90e106c841..b984ae3c31f 100644 --- a/polygon/heimdall/span_block_producers_tracker.go +++ b/polygon/heimdall/span_block_producers_tracker.go @@ -23,31 +23,32 @@ import ( "sync/atomic" "github.com/erigontech/erigon-lib/log/v3" - "github.com/erigontech/erigon/polygon/bor/borcfg" "github.com/erigontech/erigon/polygon/bor/valset" ) +type CalculateSprintNumberFunc func(uint64) uint64 + func newSpanBlockProducersTracker( logger log.Logger, - borConfig *borcfg.BorConfig, + calculateSprintNumber CalculateSprintNumberFunc, store EntityStore[*SpanBlockProducerSelection], ) *spanBlockProducersTracker { return &spanBlockProducersTracker{ - logger: logger, - borConfig: borConfig, - store: store, - newSpans: make(chan *Span), - idleSignal: make(chan struct{}), + logger: logger, + calculateSprintNumber: calculateSprintNumber, + store: store, + newSpans: make(chan *Span), + idleSignal: make(chan struct{}), } } type spanBlockProducersTracker struct { - logger log.Logger - borConfig *borcfg.BorConfig - store EntityStore[*SpanBlockProducerSelection] - newSpans chan *Span - queued atomic.Int32 - idleSignal chan struct{} + logger log.Logger + calculateSprintNumber CalculateSprintNumberFunc + store EntityStore[*SpanBlockProducerSelection] + newSpans chan *Span + queued atomic.Int32 + idleSignal chan struct{} } func (t *spanBlockProducersTracker) Run(ctx context.Context) error { @@ -141,8 +142,8 @@ func (t *spanBlockProducersTracker) ObserveSpan(ctx context.Context, newSpan *Sp return err } - spanStartSprintNum := t.borConfig.CalculateSprintNumber(lastProducerSelection.StartBlock) - spanEndSprintNum := t.borConfig.CalculateSprintNumber(lastProducerSelection.EndBlock) + spanStartSprintNum := t.calculateSprintNumber(lastProducerSelection.StartBlock) + spanEndSprintNum := t.calculateSprintNumber(lastProducerSelection.EndBlock) increments := int(spanEndSprintNum - spanStartSprintNum) if increments > 0 { producers.IncrementProposerPriority(increments) @@ -181,8 +182,8 @@ func (t *spanBlockProducersTracker) Producers(ctx context.Context, blockNum uint return nil, err } - spanStartSprintNum := t.borConfig.CalculateSprintNumber(producerSelection.StartBlock) - currentSprintNum := t.borConfig.CalculateSprintNumber(blockNum) + spanStartSprintNum := t.calculateSprintNumber(producerSelection.StartBlock) + currentSprintNum := t.calculateSprintNumber(blockNum) increments := int(currentSprintNum - spanStartSprintNum) if increments > 0 { producers.IncrementProposerPriority(increments) From 4a992a3c261e73d549dbcef7e56b982a265c5d19 Mon Sep 17 00:00:00 2001 From: Shoham Chakraborty Date: Mon, 9 Sep 2024 20:46:02 +0800 Subject: [PATCH 2/8] rpcdaemon: Integrate `heimdall.Service` (#11900) Support `bor_*` RPCs when using `polygon.sync` when rpcdaemon with datadir. --- cmd/rpcdaemon/cli/config.go | 39 ++++++++++++++----------- cmd/rpcdaemon/main.go | 7 +++-- erigon-lib/chain/chain_config.go | 1 + eth/backend.go | 4 ++- turbo/jsonrpc/bor_api.go | 16 ++++++---- turbo/jsonrpc/bor_snapshot.go | 50 ++++++++++++++++++++++---------- turbo/jsonrpc/daemon.go | 6 ++-- 7 files changed, 80 insertions(+), 43 deletions(-) diff --git a/cmd/rpcdaemon/cli/config.go b/cmd/rpcdaemon/cli/config.go index 2dfffdcc2e0..a3a66893641 100644 --- a/cmd/rpcdaemon/cli/config.go +++ b/cmd/rpcdaemon/cli/config.go @@ -55,8 +55,6 @@ import ( "github.com/erigontech/erigon-lib/kv/temporal" "github.com/erigontech/erigon-lib/log/v3" libstate "github.com/erigontech/erigon-lib/state" - "github.com/erigontech/erigon/polygon/bridge" - "github.com/erigontech/erigon/cmd/rpcdaemon/cli/httpcfg" "github.com/erigontech/erigon/cmd/rpcdaemon/graphql" "github.com/erigontech/erigon/cmd/rpcdaemon/health" @@ -75,6 +73,8 @@ import ( "github.com/erigontech/erigon/node" "github.com/erigontech/erigon/node/nodecfg" "github.com/erigontech/erigon/polygon/bor" + "github.com/erigontech/erigon/polygon/bridge" + "github.com/erigontech/erigon/polygon/heimdall" "github.com/erigontech/erigon/rpc" "github.com/erigontech/erigon/rpc/rpccfg" "github.com/erigontech/erigon/turbo/debug" @@ -323,24 +323,24 @@ func EmbeddedServices(ctx context.Context, func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger, rootCancel context.CancelFunc) ( db kv.RoDB, eth rpchelper.ApiBackend, txPool txpool.TxpoolClient, mining txpool.MiningClient, stateCache kvcache.Cache, blockReader services.FullBlockReader, engine consensus.EngineReader, - ff *rpchelper.Filters, bridgeReader bridge.ReaderService, err error) { + ff *rpchelper.Filters, bridgeReader bridge.ReaderService, heimdallReader *heimdall.Reader, err error) { if !cfg.WithDatadir && cfg.PrivateApiAddr == "" { - return nil, nil, nil, nil, nil, nil, nil, ff, nil, errors.New("either remote db or local db must be specified") + return nil, nil, nil, nil, nil, nil, nil, ff, nil, nil, errors.New("either remote db or local db must be specified") } creds, err := grpcutil.TLS(cfg.TLSCACert, cfg.TLSCertfile, cfg.TLSKeyFile) if err != nil { - return nil, nil, nil, nil, nil, nil, nil, ff, nil, fmt.Errorf("open tls cert: %w", err) + return nil, nil, nil, nil, nil, nil, nil, ff, nil, nil, fmt.Errorf("open tls cert: %w", err) } conn, err := grpcutil.Connect(creds, cfg.PrivateApiAddr) if err != nil { - return nil, nil, nil, nil, nil, nil, nil, ff, nil, fmt.Errorf("could not connect to execution service privateApi: %w", err) + return nil, nil, nil, nil, nil, nil, nil, ff, nil, nil, fmt.Errorf("could not connect to execution service privateApi: %w", err) } remoteBackendClient := remote.NewETHBACKENDClient(conn) remoteKvClient := remote.NewKVClient(conn) remoteKv, err := remotedb.NewRemote(gointerfaces.VersionFromProto(remotedbserver.KvServiceAPIVersion), logger, remoteKvClient).Open() if err != nil { - return nil, nil, nil, nil, nil, nil, nil, ff, nil, fmt.Errorf("could not connect to remoteKv: %w", err) + return nil, nil, nil, nil, nil, nil, nil, ff, nil, nil, fmt.Errorf("could not connect to remoteKv: %w", err) } // Configure DB first @@ -365,10 +365,10 @@ func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger limiter := semaphore.NewWeighted(int64(cfg.DBReadConcurrency)) rwKv, err = kv2.NewMDBX(logger).RoTxsLimiter(limiter).Path(cfg.Dirs.Chaindata).Accede().Open(ctx) if err != nil { - return nil, nil, nil, nil, nil, nil, nil, ff, nil, err + return nil, nil, nil, nil, nil, nil, nil, ff, nil, nil, err } if compatErr := checkDbCompatibility(ctx, rwKv); compatErr != nil { - return nil, nil, nil, nil, nil, nil, nil, ff, nil, compatErr + return nil, nil, nil, nil, nil, nil, nil, ff, nil, nil, compatErr } db = rwKv @@ -383,10 +383,10 @@ func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger } return nil }); err != nil { - return nil, nil, nil, nil, nil, nil, nil, ff, nil, err + return nil, nil, nil, nil, nil, nil, nil, ff, nil, nil, err } if cc == nil { - return nil, nil, nil, nil, nil, nil, nil, ff, nil, errors.New("chain config not found in db. Need start erigon at least once on this db") + return nil, nil, nil, nil, nil, nil, nil, ff, nil, nil, errors.New("chain config not found in db. Need start erigon at least once on this db") } // Configure sapshots @@ -404,7 +404,7 @@ func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger cr := rawdb.NewCanonicalReader(txNumsReader) agg, err := libstate.NewAggregator(ctx, cfg.Dirs, config3.HistoryV3AggregationStep, db, cr, logger) if err != nil { - return nil, nil, nil, nil, nil, nil, nil, ff, nil, fmt.Errorf("create aggregator: %w", err) + return nil, nil, nil, nil, nil, nil, nil, ff, nil, nil, fmt.Errorf("create aggregator: %w", err) } _ = agg.OpenFolder() //TODO: must use analog of `OptimisticReopenWithDB` @@ -455,7 +455,7 @@ func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger db, err = temporal.New(rwKv, agg) if err != nil { - return nil, nil, nil, nil, nil, nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, err } stateCache = kvcache.NewDummy() } @@ -479,7 +479,7 @@ func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger if cfg.TxPoolApiAddr != cfg.PrivateApiAddr { txpoolConn, err = grpcutil.Connect(creds, cfg.TxPoolApiAddr) if err != nil { - return nil, nil, nil, nil, nil, nil, nil, ff, nil, fmt.Errorf("could not connect to txpool api: %w", err) + return nil, nil, nil, nil, nil, nil, nil, ff, nil, nil, fmt.Errorf("could not connect to txpool api: %w", err) } } @@ -504,7 +504,12 @@ func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger stateReceiverContractAddress := cc.Bor.GetStateReceiverContract() bridgeReader, err = bridge.AssembleReader(ctx, cfg.DataDir, logger, stateReceiverContractAddress) if err != nil { - return nil, nil, nil, nil, nil, nil, nil, ff, nil, err + return nil, nil, nil, nil, nil, nil, nil, ff, nil, nil, err + } + + heimdallReader, err = heimdall.AssembleReader(ctx, cc.Bor.CalculateSprintNumber, cfg.DataDir, cfg.Dirs.Tmp, logger) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, ff, nil, nil, err } } @@ -516,7 +521,7 @@ func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger logger.Warn("[rpc] Opening Bor db", "path", borDbPath) borKv, err = kv2.NewMDBX(logger).Path(borDbPath).Label(kv.ConsensusDB).Accede().Open(ctx) if err != nil { - return nil, nil, nil, nil, nil, nil, nil, ff, nil, err + return nil, nil, nil, nil, nil, nil, nil, ff, nil, nil, err } // Skip the compatibility check, until we have a schema in erigon-lib engine = bor.NewRo(cc, borKv, blockReader, logger) @@ -554,7 +559,7 @@ func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger }() ff = rpchelper.New(ctx, cfg.RpcFiltersConfig, eth, txPool, mining, onNewSnapshot, logger) - return db, eth, txPool, mining, stateCache, blockReader, engine, ff, bridgeReader, err + return db, eth, txPool, mining, stateCache, blockReader, engine, ff, bridgeReader, heimdallReader, err } func StartRpcServer(ctx context.Context, cfg *httpcfg.HttpCfg, rpcAPI []rpc.API, logger log.Logger) error { diff --git a/cmd/rpcdaemon/main.go b/cmd/rpcdaemon/main.go index 102d5fd346b..7f0215a741e 100644 --- a/cmd/rpcdaemon/main.go +++ b/cmd/rpcdaemon/main.go @@ -40,7 +40,7 @@ func main() { cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() logger := debug.SetupCobra(cmd, "sentry") - db, backend, txPool, mining, stateCache, blockReader, engine, ff, bridgeReader, err := cli.RemoteServices(ctx, cfg, logger, rootCancel) + db, backend, txPool, mining, stateCache, blockReader, engine, ff, bridgeReader, heimdallReader, err := cli.RemoteServices(ctx, cfg, logger, rootCancel) if err != nil { if !errors.Is(err, context.Canceled) { logger.Error("Could not connect to DB", "err", err) @@ -52,8 +52,11 @@ func main() { if bridgeReader != nil { defer bridgeReader.Close() } + if heimdallReader != nil { + defer heimdallReader.Close() + } - apiList := jsonrpc.APIList(db, backend, txPool, mining, ff, stateCache, blockReader, cfg, engine, logger, bridgeReader) + apiList := jsonrpc.APIList(db, backend, txPool, mining, ff, stateCache, blockReader, cfg, engine, logger, bridgeReader, heimdallReader) rpc.PreAllocateRPCMetricLabels(apiList) if err := cli.StartRpcServer(ctx, cfg, apiList, logger); err != nil { logger.Error(err.Error()) diff --git a/erigon-lib/chain/chain_config.go b/erigon-lib/chain/chain_config.go index fa0df5d6f48..d0f2652687d 100644 --- a/erigon-lib/chain/chain_config.go +++ b/erigon-lib/chain/chain_config.go @@ -99,6 +99,7 @@ type BorConfig interface { GetNapoliBlock() *big.Int IsAhmedabad(number uint64) bool GetStateReceiverContract() string + CalculateSprintNumber(number uint64) uint64 } func (c *Config) String() string { diff --git a/eth/backend.go b/eth/backend.go index 5b7b6c91d73..105dc4265e1 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -219,6 +219,7 @@ type Ethereum struct { polygonSyncService polygonsync.Service polygonBridge bridge.PolygonBridge + heimdallService heimdall.Service stopNode func() error } @@ -555,6 +556,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger heimdallService = heimdall.AssembleService(borConfig.CalculateSprintNumber, config.HeimdallURL, dirs.DataDir, tmpdir, logger) backend.polygonBridge = polygonBridge + backend.heimdallService = heimdallService } flags.Milestone = config.WithHeimdallMilestones @@ -1021,7 +1023,7 @@ func (s *Ethereum) Init(stack *node.Node, config *ethconfig.Config, chainConfig } } - s.apiList = jsonrpc.APIList(chainKv, ethRpcClient, txPoolRpcClient, miningRpcClient, ff, stateCache, blockReader, &httpRpcCfg, s.engine, s.logger, s.polygonBridge) + s.apiList = jsonrpc.APIList(chainKv, ethRpcClient, txPoolRpcClient, miningRpcClient, ff, stateCache, blockReader, &httpRpcCfg, s.engine, s.logger, s.polygonBridge, s.heimdallService) if config.SilkwormRpcDaemon && httpRpcCfg.Enabled { interface_log_settings := silkworm.RpcInterfaceLogSettings{ diff --git a/turbo/jsonrpc/bor_api.go b/turbo/jsonrpc/bor_api.go index 9ee61ce79cb..a35c038ba4c 100644 --- a/turbo/jsonrpc/bor_api.go +++ b/turbo/jsonrpc/bor_api.go @@ -17,11 +17,11 @@ package jsonrpc import ( + "context" "fmt" "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv" - "github.com/erigontech/erigon/consensus" "github.com/erigontech/erigon/polygon/bor" "github.com/erigontech/erigon/polygon/bor/valset" @@ -43,17 +43,23 @@ type BorAPI interface { GetRootHash(start uint64, end uint64) (string, error) } +type spanProducersReader interface { + Producers(ctx context.Context, blockNum uint64) (*valset.ValidatorSet, error) +} + // BorImpl is implementation of the BorAPI interface type BorImpl struct { *BaseAPI - db kv.RoDB // the chain db + db kv.RoDB // the chain db + spanProducersReader spanProducersReader } // NewBorAPI returns BorImpl instance -func NewBorAPI(base *BaseAPI, db kv.RoDB) *BorImpl { +func NewBorAPI(base *BaseAPI, db kv.RoDB, spanProducersReader spanProducersReader) *BorImpl { return &BorImpl{ - BaseAPI: base, - db: db, + BaseAPI: base, + db: db, + spanProducersReader: spanProducersReader, } } diff --git a/turbo/jsonrpc/bor_snapshot.go b/turbo/jsonrpc/bor_snapshot.go index c6e5e89e25b..e7698e0617d 100644 --- a/turbo/jsonrpc/bor_snapshot.go +++ b/turbo/jsonrpc/bor_snapshot.go @@ -67,25 +67,40 @@ func (api *BorImpl) GetSnapshot(number *rpc.BlockNumber) (*Snapshot, error) { return nil, errUnknownBlock } - // init consensus db - bor, err := api.bor() + if api.spanProducersReader != nil { + validatorSet, err := api.spanProducersReader.Producers(ctx, header.Number.Uint64()) + if err != nil { + return nil, err + } + + snap := &Snapshot{ + Number: header.Number.Uint64(), + Hash: header.Hash(), + ValidatorSet: validatorSet, + } + + return snap, nil + } + // init consensus db + borEngine, err := api.bor() if err != nil { return nil, err } - borTx, err := bor.DB.BeginRo(ctx) + borTx, err := borEngine.DB.BeginRo(ctx) if err != nil { return nil, err } defer borTx.Rollback() + return snapshot(ctx, api, tx, borTx, header) } // GetAuthor retrieves the author a block. func (api *BorImpl) GetAuthor(blockNrOrHash *rpc.BlockNumberOrHash) (*common.Address, error) { // init consensus db - bor, err := api.bor() + borEngine, err := api.bor() if err != nil { return nil, err @@ -122,7 +137,7 @@ func (api *BorImpl) GetAuthor(blockNrOrHash *rpc.BlockNumberOrHash) (*common.Add return nil, errUnknownBlock } - author, err := bor.Author(header) + author, err := borEngine.Author(header) return &author, err } @@ -146,13 +161,13 @@ func (api *BorImpl) GetSnapshotAtHash(hash common.Hash) (*Snapshot, error) { } // init consensus db - bor, err := api.bor() + borEngine, err := api.bor() if err != nil { return nil, err } - borTx, err := bor.DB.BeginRo(ctx) + borTx, err := borEngine.DB.BeginRo(ctx) if err != nil { return nil, err } @@ -183,18 +198,23 @@ func (api *BorImpl) GetSigners(number *rpc.BlockNumber) ([]common.Address, error } // init consensus db - bor, err := api.bor() + borEngine, err := api.bor() if err != nil { return nil, err } - borTx, err := bor.DB.BeginRo(ctx) + borTx, err := borEngine.DB.BeginRo(ctx) if err != nil { return nil, err } defer borTx.Rollback() + snap, err := snapshot(ctx, api, tx, borTx, header) + if err != nil { + return nil, err + } + return snap.signers(), err } @@ -217,13 +237,13 @@ func (api *BorImpl) GetSignersAtHash(hash common.Hash) ([]common.Address, error) } // init consensus db - bor, err := api.bor() + borEngine, err := api.bor() if err != nil { return nil, err } - borTx, err := bor.DB.BeginRo(ctx) + borTx, err := borEngine.DB.BeginRo(ctx) if err != nil { return nil, err } @@ -385,13 +405,13 @@ func (api *BorImpl) GetSnapshotProposerSequence(blockNrOrHash *rpc.BlockNumberOr } // init consensus db - bor, err := api.bor() + borEngine, err := api.bor() if err != nil { return BlockSigners{}, err } - borTx, err := bor.DB.BeginRo(ctx) + borTx, err := borEngine.DB.BeginRo(ctx) if err != nil { return BlockSigners{}, err } @@ -441,7 +461,7 @@ func (api *BorImpl) GetSnapshotProposerSequence(blockNrOrHash *rpc.BlockNumberOr // GetRootHash returns the merkle root of the start to end block headers func (api *BorImpl) GetRootHash(start, end uint64) (string, error) { - bor, err := api.bor() + borEngine, err := api.bor() if err != nil { return "", err @@ -454,7 +474,7 @@ func (api *BorImpl) GetRootHash(start, end uint64) (string, error) { } defer tx.Rollback() - return bor.GetRootHash(ctx, tx, start, end) + return borEngine.GetRootHash(ctx, tx, start, end) } // Helper functions for Snapshot Type diff --git a/turbo/jsonrpc/daemon.go b/turbo/jsonrpc/daemon.go index ac8a46779be..37a4c707911 100644 --- a/turbo/jsonrpc/daemon.go +++ b/turbo/jsonrpc/daemon.go @@ -34,7 +34,7 @@ import ( func APIList(db kv.RoDB, eth rpchelper.ApiBackend, txPool txpool.TxpoolClient, mining txpool.MiningClient, filters *rpchelper.Filters, stateCache kvcache.Cache, blockReader services.FullBlockReader, cfg *httpcfg.HttpCfg, engine consensus.EngineReader, - logger log.Logger, bridgeReader bridgeReader, + logger log.Logger, bridgeReader bridgeReader, spanProducersReader spanProducersReader, ) (list []rpc.API) { base := NewBaseApi(filters, stateCache, blockReader, cfg.WithDatadir, cfg.EvmCallTimeout, engine, cfg.Dirs, bridgeReader) ethImpl := NewEthAPI(base, db, eth, txPool, mining, cfg.Gascap, cfg.Feecap, cfg.ReturnDataLimit, cfg.AllowUnprotectedTxs, cfg.MaxGetProofRewindBlockCount, cfg.WebsocketSubscribeLogsChannelSize, logger) @@ -57,10 +57,10 @@ func APIList(db kv.RoDB, eth rpchelper.ApiBackend, txPool txpool.TxpoolClient, m switch engine := engine.(type) { case *bor.Bor: - borImpl = NewBorAPI(base, db) + borImpl = NewBorAPI(base, db, spanProducersReader) case lazy: if _, ok := engine.Engine().(*bor.Bor); !engine.HasEngine() || ok { - borImpl = NewBorAPI(base, db) + borImpl = NewBorAPI(base, db, spanProducersReader) } } From 6d15c016423ed7750e1c1d6f238c71eb04dcfe34 Mon Sep 17 00:00:00 2001 From: milen <94537774+taratorio@users.noreply.github.com> Date: Mon, 9 Sep 2024 15:10:32 +0100 Subject: [PATCH 3/8] polygon/sync: block downloader to not re-insert blocks behind start (#11929) Run into an issue since we started pruning total difficulty. ``` EROR[09-09|10:58:03.057] [2/6 PolygonSync] stopping node err="parent's total difficulty not found with hash 9334099de5d77c0d56afefde9985d44f8b4416db99dfe926908d5501fa8dbd9e and height 11736178: ``` It happened for checkpoint [9703](https://heimdall-api-amoy.polygon.technology/checkpoints/9703). Our start block was in the middle of the checkpoint range which meant we have to fetch all the 8k blocks in this checkpoint to verify the checkpoint root hash when receiving blocks from the peer. The current logic will attempt to insert all these 8k blocks and it will fail with a missing parent td error because we only keep the last 1000 parent td records. This PR fixes this by enhancing the block downloader to not re-insert blocks behind the `start` block. This solves the parent td error and also is saving some unnecessary inserts on the first waypoint processing on startup. --- polygon/sync/block_downloader.go | 14 ++++++---- polygon/sync/block_downloader_test.go | 38 +++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/polygon/sync/block_downloader.go b/polygon/sync/block_downloader.go index 685dffb7f71..228bbb688c7 100644 --- a/polygon/sync/block_downloader.go +++ b/polygon/sync/block_downloader.go @@ -117,7 +117,7 @@ func (d *blockDownloader) DownloadBlocksUsingCheckpoints(ctx context.Context, st return nil, err } - return d.downloadBlocksUsingWaypoints(ctx, waypoints, d.checkpointVerifier) + return d.downloadBlocksUsingWaypoints(ctx, waypoints, d.checkpointVerifier, start) } func (d *blockDownloader) DownloadBlocksUsingMilestones(ctx context.Context, start uint64) (*types.Header, error) { @@ -126,13 +126,14 @@ func (d *blockDownloader) DownloadBlocksUsingMilestones(ctx context.Context, sta return nil, err } - return d.downloadBlocksUsingWaypoints(ctx, waypoints, d.milestoneVerifier) + return d.downloadBlocksUsingWaypoints(ctx, waypoints, d.milestoneVerifier, start) } func (d *blockDownloader) downloadBlocksUsingWaypoints( ctx context.Context, waypoints heimdall.Waypoints, verifier WaypointHeadersVerifier, + startBlockNum uint64, ) (*types.Header, error) { if len(waypoints) == 0 { return nil, nil @@ -267,9 +268,12 @@ func (d *blockDownloader) downloadBlocksUsingWaypoints( break } - if blockBatch[0].Number().Uint64() == 0 { - // we do not want to insert block 0 (genesis) - blockBatch = blockBatch[1:] + batchStart := blockBatch[0].Number().Uint64() + batchEnd := blockBatch[len(blockBatch)-1].Number().Uint64() + if batchStart <= startBlockNum && startBlockNum <= batchEnd { + // we do not want to re-insert blocks of the first waypoint if the start block + // falls in the middle of the waypoint range + blockBatch = blockBatch[startBlockNum-batchStart:] } blocks = append(blocks, blockBatch...) diff --git a/polygon/sync/block_downloader_test.go b/polygon/sync/block_downloader_test.go index 9019001eb31..f407661425a 100644 --- a/polygon/sync/block_downloader_test.go +++ b/polygon/sync/block_downloader_test.go @@ -29,9 +29,8 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" - "github.com/erigontech/erigon-lib/log/v3" - "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon/core/types" "github.com/erigontech/erigon/polygon/heimdall" "github.com/erigontech/erigon/polygon/p2p" @@ -309,6 +308,41 @@ func TestBlockDownloaderDownloadBlocksUsingCheckpoints(t *testing.T) { require.Equal(t, blocks[len(blocks)-1].Header(), tip) } +func TestBlockDownloaderDownloadBlocksUsingCheckpointsWhenStartIsInMiddleOfCheckpointRange(t *testing.T) { + test := newBlockDownloaderTest(t) + test.waypointReader.EXPECT(). + CheckpointsFromBlock(gomock.Any(), gomock.Any()). + Return(test.fakeCheckpoints(2), nil). + Times(1) + test.p2pService.EXPECT(). + ListPeersMayHaveBlockNum(gomock.Any()). + Return(test.fakePeers(2)). + Times(1) + test.p2pService.EXPECT(). + FetchHeaders(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(test.defaultFetchHeadersMock()). + Times(2) + test.p2pService.EXPECT(). + FetchBodies(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(test.defaultFetchBodiesMock()). + Times(2) + var blocks []*types.Block + test.store.EXPECT(). + InsertBlocks(gomock.Any(), gomock.Any()). + DoAndReturn(test.defaultInsertBlocksMock(&blocks)). + Times(1) + + tip, err := test.blockDownloader.DownloadBlocksUsingCheckpoints(context.Background(), 513) + require.NoError(t, err) + require.Len(t, blocks, 1536) // [513,1024] = 512 blocks + 1024 blocks from 2nd checkpoint + // check blocks are written in order + require.Equal(t, uint64(513), blocks[0].Header().Number.Uint64()) + require.Equal(t, uint64(1024), blocks[511].Header().Number.Uint64()) + require.Equal(t, uint64(1025), blocks[512].Header().Number.Uint64()) + require.Equal(t, uint64(2048), blocks[1535].Header().Number.Uint64()) + require.Equal(t, blocks[len(blocks)-1].Header(), tip) +} + func TestBlockDownloaderDownloadBlocksWhenInvalidHeadersThenPenalizePeerAndReDownload(t *testing.T) { var firstTimeInvalidReturned bool firstTimeInvalidReturnedPtr := &firstTimeInvalidReturned From 64ab751abcaa92ad930ef5dd73cdb36cb924ba03 Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Mon, 9 Sep 2024 16:40:31 +0200 Subject: [PATCH 4/8] E3: fix "minimal" and "full" pruning (#11920) --- turbo/cli/flags.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/turbo/cli/flags.go b/turbo/cli/flags.go index 03e0ae462b6..4c7822b1776 100644 --- a/turbo/cli/flags.go +++ b/turbo/cli/flags.go @@ -304,12 +304,12 @@ func ApplyFlagsForEthConfig(ctx *cli.Context, cfg *ethconfig.Config, logger log. // Full mode prunes all but the latest state if ctx.String(PruneModeFlag.Name) == "full" { mode.Blocks = prune.Distance(math.MaxUint64) - mode.History = prune.Distance(0) + mode.History = prune.Distance(config3.DefaultPruneDistance) } // Minimal mode prunes all but the latest state including blocks if ctx.String(PruneModeFlag.Name) == "minimal" { - mode.Blocks = prune.Distance(2048) // 2048 is just some blocks to allow reorgs - mode.History = prune.Distance(0) + mode.Blocks = prune.Distance(config3.DefaultPruneDistance) + mode.History = prune.Distance(config3.DefaultPruneDistance) } if err != nil { From 7789131c90f9d44013180cc8a3c95e4ed74462ed Mon Sep 17 00:00:00 2001 From: lystopad Date: Mon, 9 Sep 2024 18:11:22 +0200 Subject: [PATCH 5/8] Upgrade goreleaser-cross to 1.22.7 and alpine base image to 3.20.3 (#11931) https://github.com/erigontech/erigon/issues/11898 --- .github/workflows/ci-cd-main-branch-docker-images.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-cd-main-branch-docker-images.yml b/.github/workflows/ci-cd-main-branch-docker-images.yml index 4edc706e6d5..cdd24a6e5fd 100644 --- a/.github/workflows/ci-cd-main-branch-docker-images.yml +++ b/.github/workflows/ci-cd-main-branch-docker-images.yml @@ -3,8 +3,8 @@ run-name: "Commit id ${{ github.sha }}: CI-CD build and deploy docker images bas env: APPLICATION: "erigon" - BUILDER_IMAGE: "golang:1.22.6-alpine3.20" - TARGET_BASE_IMAGE: "alpine:3.20.2" + BUILDER_IMAGE: "golang:1.22.7-alpine3.20" + TARGET_BASE_IMAGE: "alpine:3.20.3" APP_REPO: "erigontech/erigon" CHECKOUT_REF: "main" DOCKERHUB_REPOSITORY: "erigontech/erigon" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1ea0993725e..d1b08fd368c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,8 +2,8 @@ name: Release env: APPLICATION: "erigon" - BUILDER_IMAGE: "ghcr.io/goreleaser/goreleaser-cross:v1.21.13" - DOCKER_BASE_IMAGE: "alpine:3.20.2" + BUILDER_IMAGE: "ghcr.io/goreleaser/goreleaser-cross:v1.22.7" + DOCKER_BASE_IMAGE: "alpine:3.20.3" APP_REPO: "erigontech/erigon" PACKAGE: "github.com/erigontech/erigon" DOCKERHUB_REPOSITORY: "erigontech/erigon" From 4a9b166adb2590a568b0bb2dd9d92a2ce56c6886 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Tue, 10 Sep 2024 14:20:28 +0700 Subject: [PATCH 6/8] trace_filter: return error at empty request (#11935) on empty request see error `can't find blockNumber by txnID=1235` --- erigon-lib/kv/rawdbv3/txnum.go | 16 +++++++++------- erigon-lib/kv/stream/stream.go | 3 --- erigon-lib/kv/stream/stream_test.go | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/erigon-lib/kv/rawdbv3/txnum.go b/erigon-lib/kv/rawdbv3/txnum.go index c4fd1330954..18168ca4870 100644 --- a/erigon-lib/kv/rawdbv3/txnum.go +++ b/erigon-lib/kv/rawdbv3/txnum.go @@ -237,14 +237,14 @@ func (TxNumsReader) Last(tx kv.Tx) (blockNum, txNum uint64, err error) { } defer c.Close() - lastK, lastV, err := c.Last() + k, v, err := c.Last() if err != nil { return 0, 0, err } - if lastK == nil || lastV == nil { + if k == nil || v == nil { return 0, 0, nil } - return binary.BigEndian.Uint64(lastK), binary.BigEndian.Uint64(lastV), nil + return binary.BigEndian.Uint64(k), binary.BigEndian.Uint64(v), nil } func (TxNumsReader) First(tx kv.Tx) (blockNum, txNum uint64, err error) { c, err := tx.Cursor(kv.MaxTxNum) @@ -253,14 +253,14 @@ func (TxNumsReader) First(tx kv.Tx) (blockNum, txNum uint64, err error) { } defer c.Close() - lastK, lastV, err := c.First() + k, v, err := c.First() if err != nil { return 0, 0, err } - if lastK == nil || lastV == nil { + if k == nil || v == nil { return 0, 0, nil } - return binary.BigEndian.Uint64(lastK), binary.BigEndian.Uint64(lastV), nil + return binary.BigEndian.Uint64(k), binary.BigEndian.Uint64(v), nil } // LastKey @@ -355,7 +355,9 @@ func (i *MapTxNum2BlockNumIter) Next() (txNum, blockNum uint64, txIndex int, isF return } if !ok { - return txNum, i.blockNum, txIndex, isFinalTxn, blockNumChanged, fmt.Errorf("can't find blockNumber by txnID=%d", txNum) + _lb, _lt, _ := i.txNumsReader.Last(i.tx) + _fb, _ft, _ := i.txNumsReader.First(i.tx) + return txNum, i.blockNum, txIndex, isFinalTxn, blockNumChanged, fmt.Errorf("can't find blockNumber by txnID=%d; last in db: (%d-%d, %d-%d)", txNum, _fb, _lb, _ft, _lt) } } blockNum = i.blockNum diff --git a/erigon-lib/kv/stream/stream.go b/erigon-lib/kv/stream/stream.go index ca523451e20..4cb5179273e 100644 --- a/erigon-lib/kv/stream/stream.go +++ b/erigon-lib/kv/stream/stream.go @@ -66,9 +66,6 @@ func (it *ArrStream[V]) NextBatch() ([]V, error) { } func Range[T constraints.Integer](from, to T) *RangeIter[T] { - if from == to { - to++ - } return &RangeIter[T]{i: from, to: to} } diff --git a/erigon-lib/kv/stream/stream_test.go b/erigon-lib/kv/stream/stream_test.go index bc9550390f8..ff6e1f3e708 100644 --- a/erigon-lib/kv/stream/stream_test.go +++ b/erigon-lib/kv/stream/stream_test.go @@ -213,7 +213,7 @@ func TestRange(t *testing.T) { s1 := stream.Range[uint64](1, 1) res, err := stream.ToArray[uint64](s1) require.NoError(t, err) - require.Equal(t, []uint64{1}, res) + require.Empty(t, res) }) } From 25a3c7cd74f5d96f3c29344ebcadc78bb29f8690 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Tue, 10 Sep 2024 14:43:35 +0700 Subject: [PATCH 7/8] re-introduce `StateCache` (#11925) --- turbo/jsonrpc/trace_adhoc.go | 18 ++++++++++-------- turbo/jsonrpc/trace_filtering.go | 11 ++++++----- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/turbo/jsonrpc/trace_adhoc.go b/turbo/jsonrpc/trace_adhoc.go index 42f234d201f..03cdc29f9f3 100644 --- a/turbo/jsonrpc/trace_adhoc.go +++ b/turbo/jsonrpc/trace_adhoc.go @@ -25,6 +25,7 @@ import ( "math" "strings" + "github.com/erigontech/erigon/turbo/shards" "github.com/holiman/uint256" libcommon "github.com/erigontech/erigon-lib/common" @@ -1171,12 +1172,13 @@ func (api *TraceAPIImpl) doCallMany(ctx context.Context, dbtx kv.Tx, msgs []type if err != nil { return nil, nil, err } - //stateCache := shards.NewStateCache(32, 0 /* no limit */) // this cache living only during current RPC call, but required to store state writes - cachedReader := stateReader - //cachedReader := state.NewCachedReader(stateReader, stateCache) + stateCache := shards.NewStateCache( + 32, 0 /* no limit */) // this cache living only during current RPC call, but required to store state writes + //cachedReader := stateReader + cachedReader := state.NewCachedReader(stateReader, stateCache) noop := state.NewNoopWriter() - cachedWriter := noop - //cachedWriter := state.NewCachedWriter(noop, stateCache) + //cachedWriter := noop + cachedWriter := state.NewCachedWriter(noop, stateCache) ibs := state.New(cachedReader) parentHeader, err := api.headerByRPCNumber(ctx, rpc.BlockNumber(blockNumber), dbtx) @@ -1266,9 +1268,9 @@ func (api *TraceAPIImpl) doCallMany(ctx context.Context, dbtx kv.Tx, msgs []type var cloneReader state.StateReader var sd *StateDiff if traceTypeStateDiff { - //cloneCache := stateCache.Clone() - //cloneReader = state.NewCachedReader(stateReader, cloneCache) - cloneReader = stateReader + cloneCache := stateCache.Clone() + cloneReader = state.NewCachedReader(stateReader, cloneCache) + //cloneReader = stateReader if isHistoricalStateReader { historicalStateReader.SetTxNum(baseTxNum + uint64(txIndex)) } diff --git a/turbo/jsonrpc/trace_filtering.go b/turbo/jsonrpc/trace_filtering.go index 665b75ed896..1962ecb7928 100644 --- a/turbo/jsonrpc/trace_filtering.go +++ b/turbo/jsonrpc/trace_filtering.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" + "github.com/erigontech/erigon/turbo/shards" jsoniter "github.com/json-iterator/go" "github.com/erigontech/erigon-lib/chain" @@ -582,11 +583,11 @@ func (api *TraceAPIImpl) filterV3(ctx context.Context, dbtx kv.TemporalTx, fromB } stateReader.SetTxNum(txNum) - //stateCache := shards.NewStateCache(32, 0 /* no limit */) // this cache living only during current RPC call, but required to store state writes - //cachedReader := state.NewCachedReader(stateReader, stateCache) - cachedReader := stateReader - //cachedWriter := state.NewCachedWriter(noop, stateCache) - cachedWriter := noop + stateCache := shards.NewStateCache(32, 0 /* no limit */) // this cache living only during current RPC call, but required to store state writes + cachedReader := state.NewCachedReader(stateReader, stateCache) + //cachedReader := stateReader + cachedWriter := state.NewCachedWriter(noop, stateCache) + //cachedWriter := noop vmConfig.SkipAnalysis = core.SkipAnalysis(chainConfig, blockNum) traceResult := &TraceCallResult{Trace: []*ParityTrace{}} From f3f1e67860b138d9ac0a8276217159006590cb80 Mon Sep 17 00:00:00 2001 From: lystopad Date: Tue, 10 Sep 2024 11:09:53 +0200 Subject: [PATCH 8/8] comment docker-build-check job as mentioned in the issue 11872. (#11873) comment docker-build-check job as mentioned in the issue [11872](https://github.com/erigontech/erigon/issues/11872) -- it will save us 5-6 mins time of waiting for the routine check for each workflow run (faster PR check, etc). get rid of "skip-build-cache" which is removed since v5 --- .github/workflows/ci.yml | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e86db4b0392..e14f361ca9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,6 @@ jobs: uses: golangci/golangci-lint-action@v6 with: version: v1.59.1 - skip-build-cache: true args: --help - name: Lint @@ -114,23 +113,23 @@ jobs: - name: Test erigon-lib run: cd erigon-lib && make test-no-fuzz - docker-build-check: - # don't run this on main - the PR must have run it to be merged and it misleads that this pushes the docker image - if: (${{ github.event_name == 'push' || !github.event.pull_request.draft }}) && ${{ github.ref != 'refs/heads/main' }} - runs-on: ubuntu-22.04 - - steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # fetch git tags for "git describe" +# docker-build-check: +# # don't run this on main - the PR must have run it to be merged and it misleads that this pushes the docker image +# if: (${{ github.event_name == 'push' || !github.event.pull_request.draft }}) && ${{ github.ref != 'refs/heads/main' }} +# runs-on: ubuntu-22.04 +# +# steps: +# - uses: AutoModality/action-clean@v1 +# - uses: actions/checkout@v4 +# with: +# fetch-depth: 0 # fetch git tags for "git describe" - - name: make docker (see dockerhub for image builds) - run: DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker +# - name: make docker (see dockerhub for image builds) +# run: DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker # check with root permissions, should be cached from previous build - - name: sudo make docker - run: sudo DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker +# - name: sudo make docker +# run: sudo DOCKER_TAG=thorax/erigon:ci-$GITHUB_SHA DOCKER_UID=$(id -u) DOCKER_GID=$(id -g) make docker # automated-tests: # runs-on: