diff --git a/cmd/access/node_builder/access_node_builder.go b/cmd/access/node_builder/access_node_builder.go index 1da04c85888..71539a72a6e 100644 --- a/cmd/access/node_builder/access_node_builder.go +++ b/cmd/access/node_builder/access_node_builder.go @@ -45,6 +45,7 @@ import ( "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/access/index" "github.com/onflow/flow-go/engine/access/ingestion" + "github.com/onflow/flow-go/engine/access/ingestion/tx_error_messages" pingeng "github.com/onflow/flow-go/engine/access/ping" "github.com/onflow/flow-go/engine/access/rest" "github.com/onflow/flow-go/engine/access/rest/routes" @@ -56,6 +57,7 @@ import ( "github.com/onflow/flow-go/engine/access/subscription" followereng "github.com/onflow/flow-go/engine/common/follower" "github.com/onflow/flow-go/engine/common/requester" + commonrpc "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/stop" synceng "github.com/onflow/flow-go/engine/common/synchronization" "github.com/onflow/flow-go/engine/common/version" @@ -150,6 +152,7 @@ type AccessNodeConfig struct { logTxTimeToFinalized bool logTxTimeToExecuted bool logTxTimeToFinalizedExecuted bool + logTxTimeToSealed bool retryEnabled bool rpcMetricsEnabled bool executionDataSyncEnabled bool @@ -163,7 +166,6 @@ type AccessNodeConfig struct { executionDataConfig edrequester.ExecutionDataConfig PublicNetworkConfig PublicNetworkConfig TxResultCacheSize uint - TxErrorMessagesCacheSize uint executionDataIndexingEnabled bool registersDBPath string checkpointFile string @@ -175,6 +177,7 @@ type AccessNodeConfig struct { programCacheSize uint checkPayerBalanceMode string versionControlEnabled bool + storeTxResultErrorMessages bool stopControlEnabled bool registerDBPruneThreshold uint64 } @@ -241,6 +244,7 @@ func DefaultAccessNodeConfig() *AccessNodeConfig { logTxTimeToFinalized: false, logTxTimeToExecuted: false, logTxTimeToFinalizedExecuted: false, + logTxTimeToSealed: false, pingEnabled: false, retryEnabled: false, rpcMetricsEnabled: false, @@ -248,7 +252,6 @@ func DefaultAccessNodeConfig() *AccessNodeConfig { apiRatelimits: nil, apiBurstlimits: nil, TxResultCacheSize: 0, - TxErrorMessagesCacheSize: 1000, PublicNetworkConfig: PublicNetworkConfig{ BindAddress: cmd.NotSet, Metrics: metrics.NewNoopCollector(), @@ -280,6 +283,7 @@ func DefaultAccessNodeConfig() *AccessNodeConfig { programCacheSize: 0, checkPayerBalanceMode: accessNode.Disabled.String(), versionControlEnabled: true, + storeTxResultErrorMessages: false, stopControlEnabled: false, registerDBPruneThreshold: pruner.DefaultThreshold, } @@ -302,6 +306,7 @@ type FlowAccessNodeBuilder struct { CollectionsToMarkFinalized *stdmap.Times CollectionsToMarkExecuted *stdmap.Times BlocksToMarkExecuted *stdmap.Times + BlockTransactions *stdmap.IdentifierMap TransactionMetrics *metrics.TransactionCollector TransactionValidationMetrics *metrics.TransactionValidationCollector RestMetrics *metrics.RestCollector @@ -351,6 +356,9 @@ type FlowAccessNodeBuilder struct { stateStreamGrpcServer *grpcserver.GrpcServer stateStreamBackend *statestreambackend.StateStreamBackend + nodeBackend *backend.Backend + + TxResultErrorMessagesCore *tx_error_messages.TxErrorMessagesCore } func (builder *FlowAccessNodeBuilder) buildFollowerState() *FlowAccessNodeBuilder { @@ -1234,6 +1242,10 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { "log-tx-time-to-finalized-executed", defaultConfig.logTxTimeToFinalizedExecuted, "log transaction time to finalized and executed") + flags.BoolVar(&builder.logTxTimeToSealed, + "log-tx-time-to-sealed", + defaultConfig.logTxTimeToSealed, + "log transaction time to sealed") flags.BoolVar(&builder.pingEnabled, "ping-enabled", defaultConfig.pingEnabled, @@ -1241,7 +1253,6 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { flags.BoolVar(&builder.retryEnabled, "retry-enabled", defaultConfig.retryEnabled, "whether to enable the retry mechanism at the access node level") flags.BoolVar(&builder.rpcMetricsEnabled, "rpc-metrics-enabled", defaultConfig.rpcMetricsEnabled, "whether to enable the rpc metrics") flags.UintVar(&builder.TxResultCacheSize, "transaction-result-cache-size", defaultConfig.TxResultCacheSize, "transaction result cache size.(Disabled by default i.e 0)") - flags.UintVar(&builder.TxErrorMessagesCacheSize, "transaction-error-messages-cache-size", defaultConfig.TxErrorMessagesCacheSize, "transaction error messages cache size.(By default 1000)") flags.StringVarP(&builder.nodeInfoFile, "node-info-file", "", @@ -1375,7 +1386,10 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { "tx-result-query-mode", defaultConfig.rpcConf.BackendConfig.TxResultQueryMode, "mode to use when querying transaction results. one of [local-only, execution-nodes-only(default), failover]") - + flags.BoolVar(&builder.storeTxResultErrorMessages, + "store-tx-result-error-messages", + defaultConfig.storeTxResultErrorMessages, + "whether to enable storing transaction error messages into the db") // Script Execution flags.StringVar(&builder.rpcConf.BackendConfig.ScriptExecutionMode, "script-execution-mode", @@ -1488,9 +1502,6 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { return errors.New("circuit-breaker-restore-timeout must be greater than 0") } } - if builder.TxErrorMessagesCacheSize == 0 { - return errors.New("transaction-error-messages-cache-size must be greater than 0") - } if builder.checkPayerBalanceMode != accessNode.Disabled.String() && !builder.executionDataIndexingEnabled { return errors.New("execution-data-indexing-enabled must be set if check-payer-balance is enabled") @@ -1603,7 +1614,8 @@ func (builder *FlowAccessNodeBuilder) enqueueRelayNetwork() { } func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { - var processedBlockHeight storage.ConsumerProgress + var processedFinalizedBlockHeight storage.ConsumerProgress + var processedTxErrorMessagesBlockHeight storage.ConsumerProgress if builder.executionDataSyncEnabled { builder.BuildExecutionSyncComponents() @@ -1677,6 +1689,11 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return err } + builder.BlockTransactions, err = stdmap.NewIdentifierMap(10000) + if err != nil { + return err + } + builder.BlocksToMarkExecuted, err = stdmap.NewTimes(1 * 300) // assume 1 block per second * 300 seconds return err @@ -1688,6 +1705,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.logTxTimeToFinalized, builder.logTxTimeToExecuted, builder.logTxTimeToFinalizedExecuted, + builder.logTxTimeToSealed, ) return nil }). @@ -1722,6 +1740,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.BlocksToMarkExecuted, builder.Storage.Collections, builder.Storage.Blocks, + builder.BlockTransactions, ) if err != nil { return err @@ -1799,8 +1818,8 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.TxResultsIndex = index.NewTransactionResultsIndex(builder.Reporter, builder.Storage.LightTransactionResults) return nil }). - Module("processed block height consumer progress", func(node *cmd.NodeConfig) error { - processedBlockHeight = bstorage.NewConsumerProgress(builder.DB, module.ConsumeProgressIngestionEngineBlockHeight) + Module("processed finalized block height consumer progress", func(node *cmd.NodeConfig) error { + processedFinalizedBlockHeight = bstorage.NewConsumerProgress(builder.DB, module.ConsumeProgressIngestionEngineBlockHeight) return nil }). Module("processed last full block height monotonic consumer progress", func(node *cmd.NodeConfig) error { @@ -1817,6 +1836,13 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return nil }). + Module("transaction result error messages storage", func(node *cmd.NodeConfig) error { + if builder.storeTxResultErrorMessages { + builder.Storage.TransactionResultErrorMessages = bstorage.NewTransactionResultErrorMessages(node.Metrics.Cache, node.DB, bstorage.DefaultCacheSize) + } + + return nil + }). Component("version control", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { if !builder.versionControlEnabled { noop := &module.NoopReadyDoneAware{} @@ -1944,7 +1970,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { } - nodeBackend, err := backend.New(backend.Params{ + builder.nodeBackend, err = backend.New(backend.Params{ State: node.State, CollectionRPC: builder.CollectionRPC, HistoricalAccessNodes: builder.HistoricalAccessRPCs, @@ -1954,6 +1980,7 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { Transactions: node.Storage.Transactions, ExecutionReceipts: node.Storage.Receipts, ExecutionResults: node.Storage.Results, + TxResultErrorMessages: node.Storage.TransactionResultErrorMessages, ChainID: node.RootChainID, AccessMetrics: builder.AccessMetrics, ConnFactory: connFactory, @@ -1965,7 +1992,6 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { SnapshotHistoryLimit: backend.DefaultSnapshotHistoryLimit, Communicator: backend.NewNodeCommunicator(backendConfig.CircuitBreakerConfig.Enabled), TxResultCacheSize: builder.TxResultCacheSize, - TxErrorMessagesCacheSize: builder.TxErrorMessagesCacheSize, ScriptExecutor: builder.ScriptExecutor, ScriptExecutionMode: scriptExecMode, CheckPayerBalanceMode: checkPayerBalanceMode, @@ -1997,8 +2023,8 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { builder.AccessMetrics, builder.rpcMetricsEnabled, builder.Me, - nodeBackend, - nodeBackend, + builder.nodeBackend, + builder.nodeBackend, builder.secureGrpcServer, builder.unsecureGrpcServer, builder.stateStreamBackend, @@ -2037,6 +2063,28 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return nil, fmt.Errorf("could not create requester engine: %w", err) } + preferredENIdentifiers, err := commonrpc.IdentifierList(builder.rpcConf.BackendConfig.PreferredExecutionNodeIDs) + if err != nil { + return nil, fmt.Errorf("failed to convert node id string to Flow Identifier for preferred EN map: %w", err) + } + + fixedENIdentifiers, err := commonrpc.IdentifierList(builder.rpcConf.BackendConfig.FixedExecutionNodeIDs) + if err != nil { + return nil, fmt.Errorf("failed to convert node id string to Flow Identifier for fixed EN map: %w", err) + } + + if builder.storeTxResultErrorMessages { + builder.TxResultErrorMessagesCore = tx_error_messages.NewTxErrorMessagesCore( + node.Logger, + node.State, + builder.nodeBackend, + node.Storage.Receipts, + node.Storage.TransactionResultErrorMessages, + preferredENIdentifiers, + fixedENIdentifiers, + ) + } + builder.IngestEng, err = ingestion.New( node.Logger, node.EngineRegistry, @@ -2050,8 +2098,9 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { node.Storage.Results, node.Storage.Receipts, builder.collectionExecutedMetric, - processedBlockHeight, + processedFinalizedBlockHeight, lastFullBlockHeight, + builder.TxResultErrorMessagesCore, ) if err != nil { return nil, err @@ -2069,6 +2118,31 @@ func (builder *FlowAccessNodeBuilder) Build() (cmd.Node, error) { return builder.RequestEng, nil }) + if builder.storeTxResultErrorMessages { + builder.Module("processed error messages block height consumer progress", func(node *cmd.NodeConfig) error { + processedTxErrorMessagesBlockHeight = bstorage.NewConsumerProgress( + builder.DB, + module.ConsumeProgressEngineTxErrorMessagesBlockHeight, + ) + return nil + }) + builder.Component("transaction result error messages engine", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { + engine, err := tx_error_messages.New( + node.Logger, + node.State, + node.Storage.Headers, + processedTxErrorMessagesBlockHeight, + builder.TxResultErrorMessagesCore, + ) + if err != nil { + return nil, err + } + builder.FollowerDistributor.AddOnBlockFinalizedConsumer(engine.OnFinalizedBlock) + + return engine, nil + }) + } + if builder.supportsObserver { builder.Component("public sync request handler", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { syncRequestHandler, err := synceng.NewRequestHandlerEngine( diff --git a/cmd/bootstrap/utils/key_generation.go b/cmd/bootstrap/utils/key_generation.go index fd4c8c53444..6cb662be21c 100644 --- a/cmd/bootstrap/utils/key_generation.go +++ b/cmd/bootstrap/utils/key_generation.go @@ -13,6 +13,7 @@ import ( sdk "github.com/onflow/flow-go-sdk" sdkcrypto "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/model/bootstrap" model "github.com/onflow/flow-go/model/bootstrap" diff --git a/cmd/collection/main.go b/cmd/collection/main.go index c3d1496918a..1a241ba703b 100644 --- a/cmd/collection/main.go +++ b/cmd/collection/main.go @@ -9,6 +9,7 @@ import ( client "github.com/onflow/flow-go-sdk/access/grpc" sdkcrypto "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go/admin/commands" collectionCommands "github.com/onflow/flow-go/admin/commands/collection" storageCommands "github.com/onflow/flow-go/admin/commands/storage" diff --git a/cmd/consensus/main.go b/cmd/consensus/main.go index 874385b2b3f..616e7657d10 100644 --- a/cmd/consensus/main.go +++ b/cmd/consensus/main.go @@ -12,6 +12,7 @@ import ( client "github.com/onflow/flow-go-sdk/access/grpc" "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go/cmd" "github.com/onflow/flow-go/cmd/util/cmd/common" "github.com/onflow/flow-go/consensus" diff --git a/cmd/observer/node_builder/observer_builder.go b/cmd/observer/node_builder/observer_builder.go index 21d8211daa6..ea3b9a1654a 100644 --- a/cmd/observer/node_builder/observer_builder.go +++ b/cmd/observer/node_builder/observer_builder.go @@ -19,10 +19,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" badgerds "github.com/ipfs/go-ds-badger2" - dht "github.com/libp2p/go-libp2p-kad-dht" - "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/routing" "github.com/onflow/crypto" "github.com/rs/zerolog" "github.com/spf13/pflag" @@ -92,17 +89,12 @@ import ( "github.com/onflow/flow-go/network/converter" "github.com/onflow/flow-go/network/p2p" "github.com/onflow/flow-go/network/p2p/blob" - p2pbuilder "github.com/onflow/flow-go/network/p2p/builder" - p2pbuilderconfig "github.com/onflow/flow-go/network/p2p/builder/config" "github.com/onflow/flow-go/network/p2p/cache" "github.com/onflow/flow-go/network/p2p/conduit" - p2pdht "github.com/onflow/flow-go/network/p2p/dht" "github.com/onflow/flow-go/network/p2p/keyutils" p2plogging "github.com/onflow/flow-go/network/p2p/logging" - networkingsubscription "github.com/onflow/flow-go/network/p2p/subscription" "github.com/onflow/flow-go/network/p2p/translator" "github.com/onflow/flow-go/network/p2p/unicast/protocols" - "github.com/onflow/flow-go/network/p2p/utils" "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/underlay" "github.com/onflow/flow-go/network/validator" @@ -137,8 +129,6 @@ import ( // For a node running as a standalone process, the config fields will be populated from the command line params, // while for a node running as a library, the config fields are expected to be initialized by the caller. type ObserverServiceConfig struct { - bootstrapNodeAddresses []string - bootstrapNodePublicKeys []string observerNetworkingKeyPath string bootstrapIdentities flow.IdentitySkeletonList // the identity list of bootstrap peers the node uses to discover other nodes apiRatelimits map[string]int @@ -157,6 +147,7 @@ type ObserverServiceConfig struct { logTxTimeToFinalized bool logTxTimeToExecuted bool logTxTimeToFinalizedExecuted bool + logTxTimeToSealed bool executionDataSyncEnabled bool executionDataIndexingEnabled bool executionDataDBMode string @@ -222,8 +213,6 @@ func DefaultObserverServiceConfig() *ObserverServiceConfig { rpcMetricsEnabled: false, apiRatelimits: nil, apiBurstlimits: nil, - bootstrapNodeAddresses: []string{}, - bootstrapNodePublicKeys: []string{}, observerNetworkingKeyPath: cmd.NotSet, apiTimeout: 3 * time.Second, upstreamNodeAddresses: []string{}, @@ -234,6 +223,7 @@ func DefaultObserverServiceConfig() *ObserverServiceConfig { logTxTimeToFinalized: false, logTxTimeToExecuted: false, logTxTimeToFinalizedExecuted: false, + logTxTimeToSealed: false, executionDataSyncEnabled: false, executionDataIndexingEnabled: false, executionDataDBMode: execution_data.ExecutionDataDBModeBadger.String(), @@ -333,7 +323,7 @@ func (builder *ObserverServiceBuilder) deriveBootstrapPeerIdentities() error { return nil } - ids, err := cmd.BootstrapIdentities(builder.bootstrapNodeAddresses, builder.bootstrapNodePublicKeys) + ids, err := builder.DeriveBootstrapPeerIdentities() if err != nil { return fmt.Errorf("failed to derive bootstrap peer identities: %w", err) } @@ -654,14 +644,6 @@ func (builder *ObserverServiceBuilder) extraFlags() { "observer-networking-key-path", defaultConfig.observerNetworkingKeyPath, "path to the networking key for observer") - flags.StringSliceVar(&builder.bootstrapNodeAddresses, - "bootstrap-node-addresses", - defaultConfig.bootstrapNodeAddresses, - "the network addresses of the bootstrap access node if this is an observer e.g. access-001.mainnet.flow.org:9653,access-002.mainnet.flow.org:9653") - flags.StringSliceVar(&builder.bootstrapNodePublicKeys, - "bootstrap-node-public-keys", - defaultConfig.bootstrapNodePublicKeys, - "the networking public key of the bootstrap access node if this is an observer (in the same order as the bootstrap node addresses) e.g. \"d57a5e9c5.....\",\"44ded42d....\"") flags.DurationVar(&builder.apiTimeout, "upstream-api-timeout", defaultConfig.apiTimeout, "tcp timeout for Flow API gRPC sockets to upstrem nodes") flags.StringSliceVar(&builder.upstreamNodeAddresses, "upstream-node-addresses", @@ -678,6 +660,10 @@ func (builder *ObserverServiceBuilder) extraFlags() { "log-tx-time-to-finalized-executed", defaultConfig.logTxTimeToFinalizedExecuted, "log transaction time to finalized and executed") + flags.BoolVar(&builder.logTxTimeToSealed, + "log-tx-time-to-sealed", + defaultConfig.logTxTimeToSealed, + "log transaction time to sealed") flags.BoolVar(&builder.rpcMetricsEnabled, "rpc-metrics-enabled", defaultConfig.rpcMetricsEnabled, "whether to enable the rpc metrics") flags.BoolVar(&builder.executionDataIndexingEnabled, "execution-data-indexing-enabled", @@ -1001,10 +987,10 @@ func (builder *ObserverServiceBuilder) validateParams() error { if len(builder.bootstrapIdentities) > 0 { return nil } - if len(builder.bootstrapNodeAddresses) == 0 { + if len(builder.BootstrapNodeAddresses) == 0 { return errors.New("no bootstrap node address provided") } - if len(builder.bootstrapNodeAddresses) != len(builder.bootstrapNodePublicKeys) { + if len(builder.BootstrapNodeAddresses) != len(builder.BootstrapNodePublicKeys) { return errors.New("number of bootstrap node addresses and public keys should match") } if len(builder.upstreamNodePublicKeys) > 0 && len(builder.upstreamNodeAddresses) != len(builder.upstreamNodePublicKeys) { @@ -1013,77 +999,6 @@ func (builder *ObserverServiceBuilder) validateParams() error { return nil } -// initPublicLibp2pNode creates a libp2p node for the observer service in the public (unstaked) network. -// The factory function is later passed into the initMiddleware function to eventually instantiate the p2p.LibP2PNode instance -// The LibP2P host is created with the following options: -// * DHT as client and seeded with the given bootstrap peers -// * The specified bind address as the listen address -// * The passed in private key as the libp2p key -// * No connection gater -// * No connection manager -// * No peer manager -// * Default libp2p pubsub options. -// Args: -// - networkKey: the private key to use for the libp2p node -// Returns: -// - p2p.LibP2PNode: the libp2p node -// - error: if any error occurs. Any error returned is considered irrecoverable. -func (builder *ObserverServiceBuilder) initPublicLibp2pNode(networkKey crypto.PrivateKey) (p2p.LibP2PNode, error) { - var pis []peer.AddrInfo - - for _, b := range builder.bootstrapIdentities { - pi, err := utils.PeerAddressInfo(*b) - if err != nil { - return nil, fmt.Errorf("could not extract peer address info from bootstrap identity %v: %w", b, err) - } - - pis = append(pis, pi) - } - - node, err := p2pbuilder.NewNodeBuilder( - builder.Logger, - &builder.FlowConfig.NetworkConfig.GossipSub, - &p2pbuilderconfig.MetricsConfig{ - HeroCacheFactory: builder.HeroCacheMetricsFactory(), - Metrics: builder.Metrics.Network, - }, - network.PublicNetwork, - builder.BaseConfig.BindAddr, - networkKey, - builder.SporkID, - builder.IdentityProvider, - &builder.FlowConfig.NetworkConfig.ResourceManager, - p2pbuilderconfig.PeerManagerDisableConfig(), // disable peer manager for observer node. - &p2p.DisallowListCacheConfig{ - MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, - Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), - }, - &p2pbuilderconfig.UnicastConfig{ - Unicast: builder.FlowConfig.NetworkConfig.Unicast, - }). - SetSubscriptionFilter( - networkingsubscription.NewRoleBasedFilter( - networkingsubscription.UnstakedRole, builder.IdentityProvider, - ), - ). - SetRoutingSystem(func(ctx context.Context, h host.Host) (routing.Routing, error) { - return p2pdht.NewDHT(ctx, h, protocols.FlowPublicDHTProtocolID(builder.SporkID), - builder.Logger, - builder.Metrics.Network, - p2pdht.AsClient(), - dht.BootstrapPeers(pis...), - ) - }). - Build() - if err != nil { - return nil, fmt.Errorf("could not initialize libp2p node for observer: %w", err) - } - - builder.LibP2PNode = node - - return builder.LibP2PNode, nil -} - // initObserverLocal initializes the observer's ID, network key and network address // Currently, it reads a node-info.priv.json like any other node. // TODO: read the node ID from the special bootstrap files @@ -1672,11 +1587,13 @@ func (builder *ObserverServiceBuilder) enqueuePublicNetworkInit() { builder. Component("public libp2p node", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { var err error - publicLibp2pNode, err = builder.initPublicLibp2pNode(node.NetworkKey) + publicLibp2pNode, err = builder.BuildPublicLibp2pNode(builder.BaseConfig.BindAddr, builder.bootstrapIdentities) if err != nil { - return nil, fmt.Errorf("could not create public libp2p node: %w", err) + return nil, fmt.Errorf("could not build public libp2p node: %w", err) } + builder.LibP2PNode = publicLibp2pNode + return publicLibp2pNode, nil }). Component("public network", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { @@ -1759,6 +1676,7 @@ func (builder *ObserverServiceBuilder) enqueueRPCServer() { builder.logTxTimeToFinalized, builder.logTxTimeToExecuted, builder.logTxTimeToFinalizedExecuted, + builder.logTxTimeToSealed, ) return nil }) diff --git a/cmd/scaffold.go b/cmd/scaffold.go index 5db4e75c395..a3a95cfff15 100644 --- a/cmd/scaffold.go +++ b/cmd/scaffold.go @@ -138,8 +138,8 @@ type FlowNodeBuilder struct { adminCommandBootstrapper *admin.CommandRunnerBootstrapper adminCommands map[string]func(config *NodeConfig) commands.AdminCommand componentBuilder component.ComponentManagerBuilder - bootstrapNodeAddresses []string - bootstrapNodePublicKeys []string + BootstrapNodeAddresses []string + BootstrapNodePublicKeys []string } var _ NodeBuilder = (*FlowNodeBuilder)(nil) @@ -254,13 +254,13 @@ func (fnb *FlowNodeBuilder) BaseFlags() { // observer mode allows a unstaked execution node to fetch blocks from a public staked access node, and being able to execute blocks fnb.flags.BoolVar(&fnb.BaseConfig.ObserverMode, "observer-mode", defaultConfig.ObserverMode, "whether the node is running in observer mode") - fnb.flags.StringSliceVar(&fnb.bootstrapNodePublicKeys, + fnb.flags.StringSliceVar(&fnb.BootstrapNodePublicKeys, "observer-mode-bootstrap-node-public-keys", - nil, + []string{}, "the networking public key of the bootstrap access node if this is an observer (in the same order as the bootstrap node addresses) e.g. \"d57a5e9c5.....\",\"44ded42d....\"") - fnb.flags.StringSliceVar(&fnb.bootstrapNodeAddresses, + fnb.flags.StringSliceVar(&fnb.BootstrapNodeAddresses, "observer-mode-bootstrap-node-addresses", - nil, + []string{}, "the network addresses of the bootstrap access node if this is an observer e.g. access-001.mainnet.flow.org:9653,access-002.mainnet.flow.org:9653") } @@ -413,8 +413,13 @@ func (fnb *FlowNodeBuilder) EnqueueNetworkInit() { } if fnb.ObserverMode { - // observer mode only init pulbic libp2p node - publicLibp2pNode, err := fnb.BuildPublicLibp2pNode(myAddr) + // observer mode only init public libp2p node + ids, err := fnb.DeriveBootstrapPeerIdentities() + if err != nil { + return nil, fmt.Errorf("failed to derive bootstrap peer identities: %w", err) + } + + publicLibp2pNode, err := fnb.BuildPublicLibp2pNode(myAddr, ids) if err != nil { return nil, fmt.Errorf("could not build public libp2p node: %w", err) } @@ -500,7 +505,18 @@ func (fnb *FlowNodeBuilder) HeroCacheMetricsFactory() metrics.HeroCacheMetricsFa return metrics.NewNoopHeroCacheMetricsFactory() } -// initPublicLibp2pNode creates a libp2p node for the observer service in the public (unstaked) network. +// DeriveBootstrapPeerIdentities derives the Flow Identity of the bootstrap peers from the parameters. +// These are the identities of the observers also acting as the DHT bootstrap server +func (fnb *FlowNodeBuilder) DeriveBootstrapPeerIdentities() (flow.IdentitySkeletonList, error) { + ids, err := BootstrapIdentities(fnb.BootstrapNodeAddresses, fnb.BootstrapNodePublicKeys) + if err != nil { + return nil, fmt.Errorf("failed to derive bootstrap peer identities: %w", err) + } + + return ids, nil +} + +// BuildPublicLibp2pNode creates a libp2p node for the observer service in the public (unstaked) network. // The factory function is later passed into the initMiddleware function to eventually instantiate the p2p.LibP2PNode instance // The LibP2P host is created with the following options: // * DHT as client and seeded with the given bootstrap peers @@ -515,24 +531,10 @@ func (fnb *FlowNodeBuilder) HeroCacheMetricsFactory() metrics.HeroCacheMetricsFa // Returns: // - p2p.LibP2PNode: the libp2p node // - error: if any error occurs. Any error returned is considered irrecoverable. -func (fnb *FlowNodeBuilder) BuildPublicLibp2pNode(address string) (p2p.LibP2PNode, error) { +func (fnb *FlowNodeBuilder) BuildPublicLibp2pNode(address string, bootstrapIdentities flow.IdentitySkeletonList) (p2p.LibP2PNode, error) { var pis []peer.AddrInfo - ids, err := BootstrapIdentities(fnb.bootstrapNodeAddresses, fnb.bootstrapNodePublicKeys) - if err != nil { - return nil, fmt.Errorf("could not create bootstrap identities: %w", err) - } - - for _, b := range ids { - pi, err := utils.PeerAddressInfo(*b) - if err != nil { - return nil, fmt.Errorf("could not extract peer address info from bootstrap identity %v: %w", b, err) - } - - pis = append(pis, pi) - } - - for _, b := range ids { + for _, b := range bootstrapIdentities { pi, err := utils.PeerAddressInfo(*b) if err != nil { return nil, fmt.Errorf("could not extract peer address info from bootstrap identity %v: %w", b, err) diff --git a/cmd/util/cmd/common/snapshot.go b/cmd/util/cmd/common/snapshot.go index 6e940237ea1..83b568b71b8 100644 --- a/cmd/util/cmd/common/snapshot.go +++ b/cmd/util/cmd/common/snapshot.go @@ -9,9 +9,10 @@ import ( "github.com/rs/zerolog" "github.com/sethvargo/go-retry" + "github.com/onflow/flow-go-sdk/access/grpc" + "github.com/onflow/flow-go/utils/logging" - "github.com/onflow/flow-go-sdk/access/grpc" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/state/protocol" "github.com/onflow/flow-go/state/protocol/inmem" diff --git a/cmd/util/cmd/read-badger/cmd/find-block-by-commits/main.go b/cmd/util/cmd/read-badger/cmd/find-block-by-commits/main.go index 8eec5ddee0b..b1fb1cf3106 100644 --- a/cmd/util/cmd/read-badger/cmd/find-block-by-commits/main.go +++ b/cmd/util/cmd/read-badger/cmd/find-block-by-commits/main.go @@ -80,7 +80,7 @@ func FindBlockIDByCommits( func toStateCommitments(commitsStr string) ([]flow.StateCommitment, error) { commitSlice := strings.Split(commitsStr, ",") - commits := make([]flow.StateCommitment, len(commitSlice)) + commits := make([]flow.StateCommitment, 0, len(commitSlice)) for _, c := range commitSlice { commit, err := toStateCommitment(c) if err != nil { diff --git a/cmd/util/ledger/migrations/account_based_migration_test.go b/cmd/util/ledger/migrations/account_based_migration_test.go index bd2b59b6ee6..c06a6a7f090 100644 --- a/cmd/util/ledger/migrations/account_based_migration_test.go +++ b/cmd/util/ledger/migrations/account_based_migration_test.go @@ -3,7 +3,6 @@ package migrations import ( "context" "fmt" - "testing" "github.com/onflow/cadence/common" diff --git a/cmd/util/ledger/migrations/migrator_runtime.go b/cmd/util/ledger/migrations/migrator_runtime.go index ab051393bed..35e35b065d3 100644 --- a/cmd/util/ledger/migrations/migrator_runtime.go +++ b/cmd/util/ledger/migrations/migrator_runtime.go @@ -16,6 +16,7 @@ import ( "github.com/onflow/flow-go/fvm/evm" evmStdlib "github.com/onflow/flow-go/fvm/evm/stdlib" "github.com/onflow/flow-go/fvm/storage/state" + "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/model/flow" ) @@ -111,8 +112,11 @@ func (c InterpreterMigrationRuntimeConfig) NewRuntimeInterface( } } + sc := systemcontracts.SystemContractsForChain(chainID) + return util.NewMigrationRuntimeInterface( chainID, + common.Address(sc.Crypto.Address), getCodeFunc, getContractNames, getOrLoadProgram, diff --git a/cmd/util/ledger/util/migration_runtime_interface.go b/cmd/util/ledger/util/migration_runtime_interface.go index 1043fd03ce5..12142eb15b4 100644 --- a/cmd/util/ledger/util/migration_runtime_interface.go +++ b/cmd/util/ledger/util/migration_runtime_interface.go @@ -38,6 +38,7 @@ type GerOrLoadProgramListenerFunc func( type MigrationRuntimeInterface struct { runtime.EmptyRuntimeInterface chainID flow.ChainID + CryptoContractAddress common.Address GetContractCodeFunc GetContractCodeFunc GetContractNamesFunc GetContractNamesFunc GetOrLoadProgramFunc GetOrLoadProgramFunc @@ -48,6 +49,7 @@ var _ runtime.Interface = &MigrationRuntimeInterface{} func NewMigrationRuntimeInterface( chainID flow.ChainID, + cryptoContractAddress common.Address, getCodeFunc GetContractCodeFunc, getContractNamesFunc GetContractNamesFunc, getOrLoadProgramFunc GetOrLoadProgramFunc, @@ -55,6 +57,7 @@ func NewMigrationRuntimeInterface( ) *MigrationRuntimeInterface { return &MigrationRuntimeInterface{ chainID: chainID, + CryptoContractAddress: cryptoContractAddress, GetContractCodeFunc: getCodeFunc, GetContractNamesFunc: getContractNamesFunc, GetOrLoadProgramFunc: getOrLoadProgramFunc, @@ -67,65 +70,12 @@ func (m *MigrationRuntimeInterface) ResolveLocation( location runtime.Location, ) ([]runtime.ResolvedLocation, error) { - addressLocation, isAddress := location.(common.AddressLocation) - - // if the location is not an address location, e.g. an identifier location (`import Crypto`), - // then return a single resolved location which declares all identifiers. - if !isAddress { - return []runtime.ResolvedLocation{ - { - Location: location, - Identifiers: identifiers, - }, - }, nil - } - - // if the location is an address, - // and no specific identifiers where requested in the import statement, - // then fetch all identifiers at this address - if len(identifiers) == 0 { - address := flow.Address(addressLocation.Address) - - getContractNames := m.GetContractNamesFunc - if getContractNames == nil { - return nil, errors.New("GetContractNamesFunc missing") - } - - contractNames, err := getContractNames(address) - if err != nil { - return nil, fmt.Errorf("ResolveLocation failed: %w", err) - } - - // if there are no contractNames deployed, - // then return no resolved locations - if len(contractNames) == 0 { - return nil, nil - } - - identifiers = make([]runtime.Identifier, len(contractNames)) - - for i := range identifiers { - identifiers[i] = runtime.Identifier{ - Identifier: contractNames[i], - } - } - } - - // return one resolved location per identifier. - // each resolved location is an address contract location - resolvedLocations := make([]runtime.ResolvedLocation, len(identifiers)) - for i := range resolvedLocations { - identifier := identifiers[i] - resolvedLocations[i] = runtime.ResolvedLocation{ - Location: common.AddressLocation{ - Address: addressLocation.Address, - Name: identifier.Identifier, - }, - Identifiers: []runtime.Identifier{identifier}, - } - } - - return resolvedLocations, nil + return environment.ResolveLocation( + identifiers, + location, + m.GetContractNamesFunc, + m.CryptoContractAddress, + ) } func (m *MigrationRuntimeInterface) GetCode(location runtime.Location) ([]byte, error) { diff --git a/consensus/hotstuff/verification/common.go b/consensus/hotstuff/verification/common.go index 4febede7412..04c355f4390 100644 --- a/consensus/hotstuff/verification/common.go +++ b/consensus/hotstuff/verification/common.go @@ -1,6 +1,7 @@ package verification import ( + "encoding/binary" "fmt" "github.com/onflow/crypto" @@ -8,8 +9,6 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/model/flow" - - "encoding/binary" ) // MakeVoteMessage generates the message we have to sign in order to be able diff --git a/engine/access/access_test.go b/engine/access/access_test.go index d7ea12e613c..db0e712a6aa 100644 --- a/engine/access/access_test.go +++ b/engine/access/access_test.go @@ -329,16 +329,15 @@ func (suite *Suite) TestSendTransactionToRandomCollectionNode() { connFactory.On("GetAccessAPIClient", collNode2.Address, nil).Return(col2ApiClient, &mockCloser{}, nil) bnd, err := backend.New(backend.Params{State: suite.state, - Collections: collections, - Transactions: transactions, - ChainID: suite.chainID, - AccessMetrics: metrics, - ConnFactory: connFactory, - MaxHeightRange: backend.DefaultMaxHeightRange, - Log: suite.log, - SnapshotHistoryLimit: backend.DefaultSnapshotHistoryLimit, - Communicator: backend.NewNodeCommunicator(false), - TxErrorMessagesCacheSize: 1000, + Collections: collections, + Transactions: transactions, + ChainID: suite.chainID, + AccessMetrics: metrics, + ConnFactory: connFactory, + MaxHeightRange: backend.DefaultMaxHeightRange, + Log: suite.log, + SnapshotHistoryLimit: backend.DefaultSnapshotHistoryLimit, + Communicator: backend.NewNodeCommunicator(false), }) require.NoError(suite.T(), err) @@ -642,6 +641,8 @@ func (suite *Suite) TestGetSealedTransaction() { require.NoError(suite.T(), err) blocksToMarkExecuted, err := stdmap.NewTimes(100) require.NoError(suite.T(), err) + blockTransactions, err := stdmap.NewIdentifierMap(100) + require.NoError(suite.T(), err) bnd, err := backend.New(backend.Params{State: suite.state, CollectionRPC: suite.collClient, @@ -659,7 +660,6 @@ func (suite *Suite) TestGetSealedTransaction() { Log: suite.log, SnapshotHistoryLimit: backend.DefaultSnapshotHistoryLimit, Communicator: backend.NewNodeCommunicator(false), - TxErrorMessagesCacheSize: 1000, TxResultQueryMode: backend.IndexQueryModeExecutionNodesOnly, }) require.NoError(suite.T(), err) @@ -674,6 +674,7 @@ func (suite *Suite) TestGetSealedTransaction() { blocksToMarkExecuted, collections, all.Blocks, + blockTransactions, ) require.NoError(suite.T(), err) @@ -686,7 +687,23 @@ func (suite *Suite) TestGetSealedTransaction() { // create the ingest engine processedHeight := bstorage.NewConsumerProgress(db, module.ConsumeProgressIngestionEngineBlockHeight) - ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, all.Blocks, all.Headers, collections, transactions, results, receipts, collectionExecutedMetric, processedHeight, lastFullBlockHeight) + ingestEng, err := ingestion.New( + suite.log, + suite.net, + suite.state, + suite.me, + suite.request, + all.Blocks, + all.Headers, + collections, + transactions, + results, + receipts, + collectionExecutedMetric, + processedHeight, + lastFullBlockHeight, + nil, + ) require.NoError(suite.T(), err) // 1. Assume that follower engine updated the block storage and the protocol state. The block is reported as sealed @@ -743,7 +760,6 @@ func (suite *Suite) TestGetTransactionResult() { all := util.StorageLayer(suite.T(), db) results := bstorage.NewExecutionResults(suite.metrics, db) receipts := bstorage.NewExecutionReceipts(suite.metrics, db, results, bstorage.DefaultCacheSize) - originID := unittest.IdentifierFixture() *suite.state = protocol.State{} @@ -778,6 +794,9 @@ func (suite *Suite) TestGetTransactionResult() { allIdentities := append(colIdentities, enIdentities...) finalSnapshot.On("Identities", mock.Anything).Return(allIdentities, nil) + suite.state.On("AtBlockID", blockNegativeId).Return(suite.sealedSnapshot) + suite.sealedSnapshot.On("Identities", mock.Anything).Return(allIdentities, nil) + // assume execution node returns an empty list of events suite.execClient.On("GetTransactionResult", mock.Anything, mock.Anything).Return(&execproto.GetTransactionResultResponse{ Events: nil, @@ -804,6 +823,8 @@ func (suite *Suite) TestGetTransactionResult() { require.NoError(suite.T(), err) blocksToMarkExecuted, err := stdmap.NewTimes(100) require.NoError(suite.T(), err) + blockTransactions, err := stdmap.NewIdentifierMap(100) + require.NoError(suite.T(), err) bnd, err := backend.New(backend.Params{State: suite.state, CollectionRPC: suite.collClient, @@ -821,7 +842,6 @@ func (suite *Suite) TestGetTransactionResult() { Log: suite.log, SnapshotHistoryLimit: backend.DefaultSnapshotHistoryLimit, Communicator: backend.NewNodeCommunicator(false), - TxErrorMessagesCacheSize: 1000, TxResultQueryMode: backend.IndexQueryModeExecutionNodesOnly, }) require.NoError(suite.T(), err) @@ -836,10 +856,12 @@ func (suite *Suite) TestGetTransactionResult() { blocksToMarkExecuted, collections, all.Blocks, + blockTransactions, ) require.NoError(suite.T(), err) processedHeight := bstorage.NewConsumerProgress(db, module.ConsumeProgressIngestionEngineBlockHeight) + lastFullBlockHeight, err := counters.NewPersistentStrictMonotonicCounter( bstorage.NewConsumerProgress(db, module.ConsumeProgressLastFullBlockHeight), suite.rootBlock.Height, @@ -847,7 +869,23 @@ func (suite *Suite) TestGetTransactionResult() { require.NoError(suite.T(), err) // create the ingest engine - ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, all.Blocks, all.Headers, collections, transactions, results, receipts, collectionExecutedMetric, processedHeight, lastFullBlockHeight) + ingestEng, err := ingestion.New( + suite.log, + suite.net, + suite.state, + suite.me, + suite.request, + all.Blocks, + all.Headers, + collections, + transactions, + results, + receipts, + collectionExecutedMetric, + processedHeight, + lastFullBlockHeight, + nil, + ) require.NoError(suite.T(), err) background, cancel := context.WithCancel(context.Background()) @@ -943,6 +981,7 @@ func (suite *Suite) TestGetTransactionResult() { } resp, err := handler.GetTransactionResult(context.Background(), getReq) require.Error(suite.T(), err) + require.Contains(suite.T(), err.Error(), "failed to find: transaction not in block") require.Nil(suite.T(), resp) }) @@ -1009,7 +1048,6 @@ func (suite *Suite) TestExecuteScript() { collections := bstorage.NewCollections(db, transactions) results := bstorage.NewExecutionResults(suite.metrics, db) receipts := bstorage.NewExecutionReceipts(suite.metrics, db, results, bstorage.DefaultCacheSize) - identities := unittest.IdentityListFixture(2, unittest.WithRole(flow.RoleExecution)) suite.sealedSnapshot.On("Identities", mock.Anything).Return(identities, nil) suite.finalSnapshot.On("Identities", mock.Anything).Return(identities, nil) @@ -1020,25 +1058,24 @@ func (suite *Suite) TestExecuteScript() { var err error suite.backend, err = backend.New(backend.Params{ - State: suite.state, - CollectionRPC: suite.collClient, - Blocks: all.Blocks, - Headers: all.Headers, - Collections: collections, - Transactions: transactions, - ExecutionReceipts: receipts, - ExecutionResults: results, - ChainID: suite.chainID, - AccessMetrics: suite.metrics, - ConnFactory: connFactory, - MaxHeightRange: backend.DefaultMaxHeightRange, - FixedExecutionNodeIDs: (identities.NodeIDs()).Strings(), - Log: suite.log, - SnapshotHistoryLimit: backend.DefaultSnapshotHistoryLimit, - Communicator: backend.NewNodeCommunicator(false), - ScriptExecutionMode: backend.IndexQueryModeExecutionNodesOnly, - TxErrorMessagesCacheSize: 1000, - TxResultQueryMode: backend.IndexQueryModeExecutionNodesOnly, + State: suite.state, + CollectionRPC: suite.collClient, + Blocks: all.Blocks, + Headers: all.Headers, + Collections: collections, + Transactions: transactions, + ExecutionReceipts: receipts, + ExecutionResults: results, + ChainID: suite.chainID, + AccessMetrics: suite.metrics, + ConnFactory: connFactory, + MaxHeightRange: backend.DefaultMaxHeightRange, + FixedExecutionNodeIDs: (identities.NodeIDs()).Strings(), + Log: suite.log, + SnapshotHistoryLimit: backend.DefaultSnapshotHistoryLimit, + Communicator: backend.NewNodeCommunicator(false), + ScriptExecutionMode: backend.IndexQueryModeExecutionNodesOnly, + TxResultQueryMode: backend.IndexQueryModeExecutionNodesOnly, }) require.NoError(suite.T(), err) @@ -1052,6 +1089,8 @@ func (suite *Suite) TestExecuteScript() { require.NoError(suite.T(), err) blocksToMarkExecuted, err := stdmap.NewTimes(100) require.NoError(suite.T(), err) + blockTransactions, err := stdmap.NewIdentifierMap(100) + require.NoError(suite.T(), err) collectionExecutedMetric, err := indexer.NewCollectionExecutedMetricImpl( suite.log, @@ -1061,6 +1100,7 @@ func (suite *Suite) TestExecuteScript() { blocksToMarkExecuted, collections, all.Blocks, + blockTransactions, ) require.NoError(suite.T(), err) @@ -1069,6 +1109,7 @@ func (suite *Suite) TestExecuteScript() { Once() processedHeight := bstorage.NewConsumerProgress(db, module.ConsumeProgressIngestionEngineBlockHeight) + lastFullBlockHeight, err := counters.NewPersistentStrictMonotonicCounter( bstorage.NewConsumerProgress(db, module.ConsumeProgressLastFullBlockHeight), suite.rootBlock.Height, @@ -1076,7 +1117,23 @@ func (suite *Suite) TestExecuteScript() { require.NoError(suite.T(), err) // create the ingest engine - ingestEng, err := ingestion.New(suite.log, suite.net, suite.state, suite.me, suite.request, all.Blocks, all.Headers, collections, transactions, results, receipts, collectionExecutedMetric, processedHeight, lastFullBlockHeight) + ingestEng, err := ingestion.New( + suite.log, + suite.net, + suite.state, + suite.me, + suite.request, + all.Blocks, + all.Headers, + collections, + transactions, + results, + receipts, + collectionExecutedMetric, + processedHeight, + lastFullBlockHeight, + nil, + ) require.NoError(suite.T(), err) // create another block as a predecessor of the block created earlier diff --git a/engine/access/ingestion/engine.go b/engine/access/ingestion/engine.go index b36e5598c59..0e4ac42367e 100644 --- a/engine/access/ingestion/engine.go +++ b/engine/access/ingestion/engine.go @@ -10,6 +10,7 @@ import ( "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/engine/access/ingestion/tx_error_messages" "github.com/onflow/flow-go/engine/common/fifoqueue" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/model/flow/filter" @@ -50,8 +51,9 @@ const ( // default queue capacity defaultQueueCapacity = 10_000 - // how many workers will concurrently process the tasks in the jobqueue - workersCount = 1 + // processFinalizedBlocksWorkersCount defines the number of workers that + // concurrently process finalized blocks in the job queue. + processFinalizedBlocksWorkersCount = 1 // ensure blocks are processed sequentially by jobqueue searchAhead = 1 @@ -77,10 +79,12 @@ type Engine struct { executionReceiptsQueue engine.MessageStore // Job queue finalizedBlockConsumer *jobqueue.ComponentConsumer - // Notifier for queue consumer finalizedBlockNotifier engine.Notifier + // txResultErrorMessagesChan is used to fetch and store transaction result error messages for blocks + txResultErrorMessagesChan chan flow.Identifier + log zerolog.Logger // used to log relevant actions with context state protocol.State // used to access the protocol state me module.Local // used to access local node information @@ -99,6 +103,8 @@ type Engine struct { lastFullBlockHeight *counters.PersistentStrictMonotonicCounter // metrics collectionExecutedMetric module.CollectionExecutedMetric + + txErrorMessagesCore *tx_error_messages.TxErrorMessagesCore } var _ network.MessageProcessor = (*Engine)(nil) @@ -119,8 +125,9 @@ func New( executionResults storage.ExecutionResults, executionReceipts storage.ExecutionReceipts, collectionExecutedMetric module.CollectionExecutedMetric, - processedHeight storage.ConsumerProgress, + finalizedProcessedHeight storage.ConsumerProgress, lastFullBlockHeight *counters.PersistentStrictMonotonicCounter, + txErrorMessagesCore *tx_error_messages.TxErrorMessagesCore, ) (*Engine, error) { executionReceiptsRawQueue, err := fifoqueue.NewFifoQueue(defaultQueueCapacity) if err != nil { @@ -162,8 +169,10 @@ func New( // queue / notifier for execution receipts executionReceiptsNotifier: engine.NewNotifier(), + txResultErrorMessagesChan: make(chan flow.Identifier, 1), executionReceiptsQueue: executionReceiptsQueue, messageHandler: messageHandler, + txErrorMessagesCore: txErrorMessagesCore, } // jobqueue Jobs object that tracks finalized blocks by height. This is used by the finalizedBlockConsumer @@ -172,7 +181,7 @@ func New( defaultIndex, err := e.defaultProcessedIndex() if err != nil { - return nil, fmt.Errorf("could not read default processed index: %w", err) + return nil, fmt.Errorf("could not read default finalized processed index: %w", err) } // create a jobqueue that will process new available finalized block. The `finalizedBlockNotifier` is used to @@ -180,11 +189,11 @@ func New( e.finalizedBlockConsumer, err = jobqueue.NewComponentConsumer( e.log.With().Str("module", "ingestion_block_consumer").Logger(), e.finalizedBlockNotifier.Channel(), - processedHeight, + finalizedProcessedHeight, finalizedBlockReader, defaultIndex, e.processFinalizedBlockJob, - workersCount, + processFinalizedBlocksWorkersCount, searchAhead, ) if err != nil { @@ -192,11 +201,20 @@ func New( } // Add workers - e.ComponentManager = component.NewComponentManagerBuilder(). + builder := component.NewComponentManagerBuilder(). AddWorker(e.processBackground). AddWorker(e.processExecutionReceipts). - AddWorker(e.runFinalizedBlockConsumer). - Build() + AddWorker(e.runFinalizedBlockConsumer) + + // If txErrorMessagesCore is provided, add a worker responsible for processing + // transaction result error messages by receipts. This worker listens for blocks + // containing execution receipts and processes any associated transaction result + // error messages. The worker is added only when error message processing is enabled. + if txErrorMessagesCore != nil { + builder.AddWorker(e.processTransactionResultErrorMessagesByReceipts) + } + + e.ComponentManager = builder.Build() // register engine with the execution receipt provider _, err = net.Register(channels.ReceiveReceipts, e) @@ -209,7 +227,7 @@ func New( // defaultProcessedIndex returns the last finalized block height from the protocol state. // -// The BlockConsumer utilizes this return height to fetch and consume block jobs from +// The finalizedBlockConsumer utilizes this return height to fetch and consume block jobs from // jobs queue the first time it initializes. // // No errors are expected during normal operation. @@ -337,6 +355,42 @@ func (e *Engine) processAvailableExecutionReceipts(ctx context.Context) error { if err := e.handleExecutionReceipt(msg.OriginID, receipt); err != nil { return err } + + // Notify to fetch and store transaction result error messages for the block. + // If txErrorMessagesCore is enabled, the receipt's BlockID is sent to trigger + // transaction error message processing. This step is skipped if error message + // storage is not enabled. + if e.txErrorMessagesCore != nil { + e.txResultErrorMessagesChan <- receipt.BlockID + } + } +} + +// processTransactionResultErrorMessagesByReceipts handles error messages related to transaction +// results by reading from the error messages channel and processing them accordingly. +// +// This function listens for messages on the txResultErrorMessagesChan channel and +// processes each transaction result error message as it arrives. +// +// No errors are expected during normal operation. +func (e *Engine) processTransactionResultErrorMessagesByReceipts(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + ready() + + for { + select { + case <-ctx.Done(): + return + case blockID := <-e.txResultErrorMessagesChan: + err := e.txErrorMessagesCore.HandleTransactionResultErrorMessages(ctx, blockID) + if err != nil { + // TODO: we should revisit error handling here. + // Errors that come from querying the EN and possibly ExecutionNodesForBlockID should be logged and + // retried later, while others should cause an exception. + e.log.Error(). + Err(err). + Msg("error encountered while processing transaction result error messages by receipts") + } + } } } diff --git a/engine/access/ingestion/engine_test.go b/engine/access/ingestion/engine_test.go index d93056bd80c..2f5b0169b34 100644 --- a/engine/access/ingestion/engine_test.go +++ b/engine/access/ingestion/engine_test.go @@ -47,18 +47,19 @@ type Suite struct { params *protocol.Params } - me *modulemock.Local - net *mocknetwork.Network - request *modulemock.Requester - obsIdentity *flow.Identity - provider *mocknetwork.Engine - blocks *storage.Blocks - headers *storage.Headers - collections *storage.Collections - transactions *storage.Transactions - receipts *storage.ExecutionReceipts - results *storage.ExecutionResults - seals *storage.Seals + me *modulemock.Local + net *mocknetwork.Network + request *modulemock.Requester + obsIdentity *flow.Identity + provider *mocknetwork.Engine + blocks *storage.Blocks + headers *storage.Headers + collections *storage.Collections + transactions *storage.Transactions + receipts *storage.ExecutionReceipts + results *storage.ExecutionResults + seals *storage.Seals + conduit *mocknetwork.Conduit downloader *downloadermock.Downloader sealedBlock *flow.Header @@ -102,7 +103,6 @@ func (s *Suite) SetupTest() { s.proto.params = new(protocol.Params) s.finalizedBlock = unittest.BlockHeaderFixture(unittest.WithHeaderHeight(0)) s.proto.state.On("Identity").Return(s.obsIdentity, nil) - s.proto.state.On("Final").Return(s.proto.snapshot, nil) s.proto.state.On("Params").Return(s.proto.params) s.proto.snapshot.On("Head").Return( func() *flow.Header { @@ -119,7 +119,6 @@ func (s *Suite) SetupTest() { Return(conduit, nil). Once() s.request = modulemock.NewRequester(s.T()) - s.provider = mocknetwork.NewEngine(s.T()) s.blocks = storage.NewBlocks(s.T()) s.headers = storage.NewHeaders(s.T()) @@ -133,6 +132,8 @@ func (s *Suite) SetupTest() { require.NoError(s.T(), err) blocksToMarkExecuted, err := stdmap.NewTimes(100) require.NoError(s.T(), err) + blockTransactions, err := stdmap.NewIdentifierMap(100) + require.NoError(s.T(), err) s.proto.state.On("Identity").Return(s.obsIdentity, nil) s.proto.state.On("Params").Return(s.proto.params) @@ -166,6 +167,10 @@ func (s *Suite) SetupTest() { ).Maybe() s.proto.state.On("Final").Return(s.proto.snapshot, nil) + // Mock the finalized root block header with height 0. + header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(0)) + s.proto.params.On("FinalizedRoot").Return(header, nil) + s.collectionExecutedMetric, err = indexer.NewCollectionExecutedMetricImpl( s.log, metrics.NewNoopCollector(), @@ -174,6 +179,7 @@ func (s *Suite) SetupTest() { blocksToMarkExecuted, s.collections, s.blocks, + blockTransactions, ) require.NoError(s.T(), err) } @@ -189,7 +195,24 @@ func (s *Suite) initIngestionEngine(ctx irrecoverable.SignalerContext) *Engine { ) require.NoError(s.T(), err) - eng, err := New(s.log, s.net, s.proto.state, s.me, s.request, s.blocks, s.headers, s.collections, s.transactions, s.results, s.receipts, s.collectionExecutedMetric, processedHeight, s.lastFullBlockHeight) + eng, err := New( + s.log, + s.net, + s.proto.state, + s.me, + s.request, + s.blocks, + s.headers, + s.collections, + s.transactions, + s.results, + s.receipts, + s.collectionExecutedMetric, + processedHeight, + s.lastFullBlockHeight, + nil, + ) + require.NoError(s.T(), err) eng.ComponentManager.Start(ctx) diff --git a/engine/access/ingestion/tx_error_messages/tx_error_messages_core.go b/engine/access/ingestion/tx_error_messages/tx_error_messages_core.go new file mode 100644 index 00000000000..88f3b93ec5d --- /dev/null +++ b/engine/access/ingestion/tx_error_messages/tx_error_messages_core.go @@ -0,0 +1,144 @@ +package tx_error_messages + +import ( + "context" + "fmt" + + execproto "github.com/onflow/flow/protobuf/go/flow/execution" + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/engine/access/rpc/backend" + commonrpc "github.com/onflow/flow-go/engine/common/rpc" + "github.com/onflow/flow-go/engine/common/rpc/convert" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" +) + +// TxErrorMessagesCore is responsible for managing transaction result error messages +// It handles both storage and retrieval of error messages +// from execution nodes. +type TxErrorMessagesCore struct { + log zerolog.Logger // used to log relevant actions with context + state protocol.State // used to access the protocol state + + backend *backend.Backend + + executionReceipts storage.ExecutionReceipts + transactionResultErrorMessages storage.TransactionResultErrorMessages + + preferredExecutionNodeIDs flow.IdentifierList + fixedExecutionNodeIDs flow.IdentifierList +} + +// NewTxErrorMessagesCore creates a new instance of TxErrorMessagesCore. +func NewTxErrorMessagesCore( + log zerolog.Logger, + state protocol.State, + backend *backend.Backend, + executionReceipts storage.ExecutionReceipts, + transactionResultErrorMessages storage.TransactionResultErrorMessages, + preferredExecutionNodeIDs flow.IdentifierList, + fixedExecutionNodeIDs flow.IdentifierList, +) *TxErrorMessagesCore { + return &TxErrorMessagesCore{ + log: log.With().Str("module", "tx_error_messages_core").Logger(), + state: state, + backend: backend, + executionReceipts: executionReceipts, + transactionResultErrorMessages: transactionResultErrorMessages, + preferredExecutionNodeIDs: preferredExecutionNodeIDs, + fixedExecutionNodeIDs: fixedExecutionNodeIDs, + } +} + +// HandleTransactionResultErrorMessages processes transaction result error messages for a given block ID. +// It retrieves error messages from the backend if they do not already exist in storage. +// +// The function first checks if error messages for the given block ID are already present in storage. +// If they are not, it fetches the messages from execution nodes and stores them. +// +// Parameters: +// - ctx: The context for managing cancellation and deadlines during the operation. +// - blockID: The identifier of the block for which transaction result error messages need to be processed. +// +// No errors are expected during normal operation. +func (c *TxErrorMessagesCore) HandleTransactionResultErrorMessages(ctx context.Context, blockID flow.Identifier) error { + exists, err := c.transactionResultErrorMessages.Exists(blockID) + if err != nil { + return fmt.Errorf("could not check existance of transaction result error messages: %w", err) + } + + if exists { + return nil + } + + // retrieves error messages from the backend if they do not already exist in storage + execNodes, err := commonrpc.ExecutionNodesForBlockID( + ctx, + blockID, + c.executionReceipts, + c.state, + c.log, + c.preferredExecutionNodeIDs, + c.fixedExecutionNodeIDs, + ) + if err != nil { + c.log.Error().Err(err).Msg(fmt.Sprintf("failed to find execution nodes for block id: %s", blockID)) + return fmt.Errorf("could not find execution nodes for block: %w", err) + } + + req := &execproto.GetTransactionErrorMessagesByBlockIDRequest{ + BlockId: convert.IdentifierToMessage(blockID), + } + + c.log.Debug(). + Msgf("transaction error messages for block %s are being downloaded", blockID) + + resp, execNode, err := c.backend.GetTransactionErrorMessagesFromAnyEN(ctx, execNodes, req) + if err != nil { + c.log.Error().Err(err).Msg("failed to get transaction error messages from execution nodes") + return err + } + + if len(resp) > 0 { + err = c.storeTransactionResultErrorMessages(blockID, resp, execNode) + if err != nil { + return fmt.Errorf("could not store error messages (block: %s): %w", blockID, err) + } + } + + return nil +} + +// storeTransactionResultErrorMessages stores the transaction result error messages for a given block ID. +// +// Parameters: +// - blockID: The identifier of the block for which the error messages are to be stored. +// - errorMessagesResponses: A slice of responses containing the error messages to be stored. +// - execNode: The execution node associated with the error messages. +// +// No errors are expected during normal operation. +func (c *TxErrorMessagesCore) storeTransactionResultErrorMessages( + blockID flow.Identifier, + errorMessagesResponses []*execproto.GetTransactionErrorMessagesResponse_Result, + execNode *flow.IdentitySkeleton, +) error { + errorMessages := make([]flow.TransactionResultErrorMessage, 0, len(errorMessagesResponses)) + for _, value := range errorMessagesResponses { + errorMessage := flow.TransactionResultErrorMessage{ + ErrorMessage: value.ErrorMessage, + TransactionID: convert.MessageToIdentifier(value.TransactionId), + Index: value.Index, + ExecutorID: execNode.NodeID, + } + errorMessages = append(errorMessages, errorMessage) + } + + err := c.transactionResultErrorMessages.Store(blockID, errorMessages) + if err != nil { + return fmt.Errorf("failed to store transaction error messages: %w", err) + } + + return nil +} diff --git a/engine/access/ingestion/tx_error_messages/tx_error_messages_core_test.go b/engine/access/ingestion/tx_error_messages/tx_error_messages_core_test.go new file mode 100644 index 00000000000..9d5d6466d97 --- /dev/null +++ b/engine/access/ingestion/tx_error_messages/tx_error_messages_core_test.go @@ -0,0 +1,332 @@ +package tx_error_messages + +import ( + "context" + "fmt" + "os" + "testing" + + execproto "github.com/onflow/flow/protobuf/go/flow/execution" + "github.com/rs/zerolog" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + accessmock "github.com/onflow/flow-go/engine/access/mock" + "github.com/onflow/flow-go/engine/access/rpc/backend" + connectionmock "github.com/onflow/flow-go/engine/access/rpc/connection/mock" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/irrecoverable" + protocol "github.com/onflow/flow-go/state/protocol/mock" + storage "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/unittest" +) + +const expectedErrorMsg = "expected test error" + +type TxErrorMessagesCoreSuite struct { + suite.Suite + + log zerolog.Logger + proto struct { + state *protocol.FollowerState + snapshot *protocol.Snapshot + params *protocol.Params + } + + receipts *storage.ExecutionReceipts + txErrorMessages *storage.TransactionResultErrorMessages + + enNodeIDs flow.IdentityList + execClient *accessmock.ExecutionAPIClient + connFactory *connectionmock.ConnectionFactory + + blockMap map[uint64]*flow.Block + rootBlock flow.Block + finalizedBlock *flow.Header + + ctx context.Context + cancel context.CancelFunc +} + +func TestTxErrorMessagesCore(t *testing.T) { + suite.Run(t, new(TxErrorMessagesCoreSuite)) +} + +// TearDownTest stops the engine and cleans up the db +func (s *TxErrorMessagesCoreSuite) TearDownTest() { + s.cancel() +} + +type mockCloser struct{} + +func (mc *mockCloser) Close() error { return nil } + +func (s *TxErrorMessagesCoreSuite) SetupTest() { + s.log = zerolog.New(os.Stderr) + s.ctx, s.cancel = context.WithCancel(context.Background()) + // mock out protocol state + s.proto.state = protocol.NewFollowerState(s.T()) + s.proto.snapshot = protocol.NewSnapshot(s.T()) + s.proto.params = protocol.NewParams(s.T()) + s.execClient = accessmock.NewExecutionAPIClient(s.T()) + s.connFactory = connectionmock.NewConnectionFactory(s.T()) + s.receipts = storage.NewExecutionReceipts(s.T()) + s.txErrorMessages = storage.NewTransactionResultErrorMessages(s.T()) + + s.rootBlock = unittest.BlockFixture() + s.rootBlock.Header.Height = 0 + s.finalizedBlock = unittest.BlockWithParentFixture(s.rootBlock.Header).Header + + s.proto.state.On("Params").Return(s.proto.params) + + // Mock the finalized root block header with height 0. + s.proto.params.On("FinalizedRoot").Return(s.rootBlock.Header, nil) + + s.proto.snapshot.On("Head").Return( + func() *flow.Header { + return s.finalizedBlock + }, + nil, + ).Maybe() + s.proto.state.On("Final").Return(s.proto.snapshot, nil) + + // Create identities for 1 execution nodes. + s.enNodeIDs = unittest.IdentityListFixture(1, unittest.WithRole(flow.RoleExecution)) +} + +// TestHandleTransactionResultErrorMessages checks that transaction result error messages +// are properly fetched from the execution nodes, processed, and stored in the protocol database. +func (s *TxErrorMessagesCoreSuite) TestHandleTransactionResultErrorMessages() { + irrecoverableCtx := irrecoverable.NewMockSignalerContext(s.T(), s.ctx) + + block := unittest.BlockWithParentFixture(s.finalizedBlock) + blockId := block.ID() + + s.connFactory.On("GetExecutionAPIClient", mock.Anything).Return(s.execClient, &mockCloser{}, nil) + + // Mock the protocol snapshot to return fixed execution node IDs. + setupReceiptsForBlock(s.receipts, block, s.enNodeIDs.NodeIDs()[0]) + s.proto.snapshot.On("Identities", mock.Anything).Return(s.enNodeIDs, nil) + s.proto.state.On("AtBlockID", blockId).Return(s.proto.snapshot).Once() + + // Create mock transaction results with a mix of failed and non-failed transactions. + resultsByBlockID := mockTransactionResultsByBlock(5) + + // Prepare a request to fetch transaction error messages by block ID from execution nodes. + exeEventReq := &execproto.GetTransactionErrorMessagesByBlockIDRequest{ + BlockId: blockId[:], + } + + s.execClient.On("GetTransactionErrorMessagesByBlockID", mock.Anything, exeEventReq). + Return(createTransactionErrorMessagesResponse(resultsByBlockID), nil). + Once() + + // Mock the txErrorMessages storage to confirm that error messages do not exist yet. + s.txErrorMessages.On("Exists", blockId). + Return(false, nil).Once() + + // Prepare the expected transaction error messages that should be stored. + expectedStoreTxErrorMessages := createExpectedTxErrorMessages(resultsByBlockID, s.enNodeIDs.NodeIDs()[0]) + + // Mock the storage of the fetched error messages into the protocol database. + s.txErrorMessages.On("Store", blockId, expectedStoreTxErrorMessages). + Return(nil).Once() + + core := s.initCore() + err := core.HandleTransactionResultErrorMessages(irrecoverableCtx, blockId) + require.NoError(s.T(), err) + + // Verify that the mock expectations for storing the error messages were met. + s.txErrorMessages.AssertExpectations(s.T()) + + // Now simulate the second try when the error messages already exist in storage. + // Mock the txErrorMessages storage to confirm that error messages exist. + s.txErrorMessages.On("Exists", blockId). + Return(true, nil).Once() + err = core.HandleTransactionResultErrorMessages(irrecoverableCtx, blockId) + require.NoError(s.T(), err) + + // Verify that the mock expectations for storing the error messages were not met. + s.txErrorMessages.AssertExpectations(s.T()) + s.execClient.AssertExpectations(s.T()) +} + +// TestHandleTransactionResultErrorMessages_ErrorCases tests the error handling of +// the HandleTransactionResultErrorMessages function in the following cases: +// +// 1. Execution node fetch error: When fetching transaction error messages from the execution node fails, +// the function should return an appropriate error and no further actions should be taken. +// 2. Storage store error after fetching results: When fetching transaction error messages succeeds, +// but storing them in the storage fails, the function should return an error and no further actions should be taken. +func (s *TxErrorMessagesCoreSuite) TestHandleTransactionResultErrorMessages_ErrorCases() { + irrecoverableCtx := irrecoverable.NewMockSignalerContext(s.T(), s.ctx) + + block := unittest.BlockWithParentFixture(s.finalizedBlock) + blockId := block.ID() + + s.connFactory.On("GetExecutionAPIClient", mock.Anything).Return(s.execClient, &mockCloser{}, nil) + + // Mock the protocol snapshot to return fixed execution node IDs. + setupReceiptsForBlock(s.receipts, block, s.enNodeIDs.NodeIDs()[0]) + s.proto.snapshot.On("Identities", mock.Anything).Return(s.enNodeIDs, nil) + s.proto.state.On("AtBlockID", blockId).Return(s.proto.snapshot) + + s.Run("Execution node fetch error", func() { + // Mock the txErrorMessages storage to confirm that error messages do not exist yet. + s.txErrorMessages.On("Exists", blockId).Return(false, nil).Once() + + // Simulate an error when fetching transaction error messages from the execution node. + exeEventReq := &execproto.GetTransactionErrorMessagesByBlockIDRequest{ + BlockId: blockId[:], + } + s.execClient.On("GetTransactionErrorMessagesByBlockID", mock.Anything, exeEventReq). + Return(nil, fmt.Errorf("execution node fetch error")).Once() + + core := s.initCore() + err := core.HandleTransactionResultErrorMessages(irrecoverableCtx, blockId) + + // Assert that the function returns an error due to the client fetch error. + require.Error(s.T(), err) + require.Contains(s.T(), err.Error(), "execution node fetch error") + + // Ensure that no further steps are taken after the client fetch error. + s.txErrorMessages.AssertNotCalled(s.T(), "Store", mock.Anything, mock.Anything) + }) + + s.Run("Storage error after fetching results", func() { + // Simulate successful fetching of transaction error messages but error in storing them. + + // Mock the txErrorMessages storage to confirm that error messages do not exist yet. + s.txErrorMessages.On("Exists", blockId).Return(false, nil).Once() + + // Create mock transaction results with a mix of failed and non-failed transactions. + resultsByBlockID := mockTransactionResultsByBlock(5) + + // Prepare a request to fetch transaction error messages by block ID from execution nodes. + exeEventReq := &execproto.GetTransactionErrorMessagesByBlockIDRequest{ + BlockId: blockId[:], + } + s.execClient.On("GetTransactionErrorMessagesByBlockID", mock.Anything, exeEventReq). + Return(createTransactionErrorMessagesResponse(resultsByBlockID), nil).Once() + + // Simulate an error when attempting to store the fetched transaction error messages in storage. + expectedStoreTxErrorMessages := createExpectedTxErrorMessages(resultsByBlockID, s.enNodeIDs.NodeIDs()[0]) + s.txErrorMessages.On("Store", blockId, expectedStoreTxErrorMessages). + Return(fmt.Errorf("storage error")).Once() + + core := s.initCore() + err := core.HandleTransactionResultErrorMessages(irrecoverableCtx, blockId) + + // Assert that the function returns an error due to the store error. + require.Error(s.T(), err) + require.Contains(s.T(), err.Error(), "storage error") + + // Ensure that storage existence check and transaction fetch were called before the store error. + s.txErrorMessages.AssertCalled(s.T(), "Exists", blockId) + s.execClient.AssertCalled(s.T(), "GetTransactionErrorMessagesByBlockID", mock.Anything, exeEventReq) + }) +} + +// initCore create new instance of transaction error messages core. +func (s *TxErrorMessagesCoreSuite) initCore() *TxErrorMessagesCore { + // Initialize the backend + backend, err := backend.New(backend.Params{ + State: s.proto.state, + ExecutionReceipts: s.receipts, + ConnFactory: s.connFactory, + MaxHeightRange: backend.DefaultMaxHeightRange, + FixedExecutionNodeIDs: s.enNodeIDs.NodeIDs().Strings(), + Log: s.log, + SnapshotHistoryLimit: backend.DefaultSnapshotHistoryLimit, + Communicator: backend.NewNodeCommunicator(false), + ScriptExecutionMode: backend.IndexQueryModeExecutionNodesOnly, + TxResultQueryMode: backend.IndexQueryModeExecutionNodesOnly, + ChainID: flow.Testnet, + }) + require.NoError(s.T(), err) + + core := NewTxErrorMessagesCore( + s.log, + s.proto.state, + backend, + s.receipts, + s.txErrorMessages, + s.enNodeIDs.NodeIDs(), + nil, + ) + return core +} + +// createExpectedTxErrorMessages creates a list of expected transaction error messages based on transaction results +func createExpectedTxErrorMessages(resultsByBlockID []flow.LightTransactionResult, executionNode flow.Identifier) []flow.TransactionResultErrorMessage { + // Prepare the expected transaction error messages that should be stored. + var expectedStoreTxErrorMessages []flow.TransactionResultErrorMessage + + for i, result := range resultsByBlockID { + if result.Failed { + errMsg := fmt.Sprintf("%s.%s", expectedErrorMsg, result.TransactionID) + + expectedStoreTxErrorMessages = append(expectedStoreTxErrorMessages, + flow.TransactionResultErrorMessage{ + TransactionID: result.TransactionID, + ErrorMessage: errMsg, + Index: uint32(i), + ExecutorID: executionNode, + }) + } + } + + return expectedStoreTxErrorMessages +} + +// mockTransactionResultsByBlock create mock transaction results with a mix of failed and non-failed transactions. +func mockTransactionResultsByBlock(count int) []flow.LightTransactionResult { + // Create mock transaction results with a mix of failed and non-failed transactions. + resultsByBlockID := make([]flow.LightTransactionResult, 0) + for i := 0; i < count; i++ { + resultsByBlockID = append(resultsByBlockID, flow.LightTransactionResult{ + TransactionID: unittest.IdentifierFixture(), + Failed: i%2 == 0, // create a mix of failed and non-failed transactions + ComputationUsed: 0, + }) + } + + return resultsByBlockID +} + +// setupReceiptsForBlock sets up mock execution receipts for a block and returns the receipts along +// with the identities of the execution nodes that processed them. +func setupReceiptsForBlock(receipts *storage.ExecutionReceipts, block *flow.Block, eNodeID flow.Identifier) { + receipt1 := unittest.ReceiptForBlockFixture(block) + receipt1.ExecutorID = eNodeID + receipt2 := unittest.ReceiptForBlockFixture(block) + receipt2.ExecutorID = eNodeID + receipt1.ExecutionResult = receipt2.ExecutionResult + + receiptsList := flow.ExecutionReceiptList{receipt1, receipt2} + + receipts. + On("ByBlockID", block.ID()). + Return(func(flow.Identifier) flow.ExecutionReceiptList { + return receiptsList + }, nil) +} + +// createTransactionErrorMessagesResponse create TransactionErrorMessagesResponse from execution node based on results. +func createTransactionErrorMessagesResponse(resultsByBlockID []flow.LightTransactionResult) *execproto.GetTransactionErrorMessagesResponse { + exeErrMessagesResp := &execproto.GetTransactionErrorMessagesResponse{} + + for i, result := range resultsByBlockID { + if result.Failed { + errMsg := fmt.Sprintf("%s.%s", expectedErrorMsg, result.TransactionID) + exeErrMessagesResp.Results = append(exeErrMessagesResp.Results, &execproto.GetTransactionErrorMessagesResponse_Result{ + TransactionId: result.TransactionID[:], + ErrorMessage: errMsg, + Index: uint32(i), + }) + } + } + + return exeErrMessagesResp +} diff --git a/engine/access/ingestion/tx_error_messages/tx_error_messages_engine.go b/engine/access/ingestion/tx_error_messages/tx_error_messages_engine.go new file mode 100644 index 00000000000..cdd65bdc0b3 --- /dev/null +++ b/engine/access/ingestion/tx_error_messages/tx_error_messages_engine.go @@ -0,0 +1,181 @@ +package tx_error_messages + +import ( + "context" + "fmt" + "time" + + "github.com/rs/zerolog" + "github.com/sethvargo/go-retry" + + "github.com/onflow/flow-go/consensus/hotstuff/model" + "github.com/onflow/flow-go/engine" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/component" + "github.com/onflow/flow-go/module/irrecoverable" + "github.com/onflow/flow-go/module/jobqueue" + "github.com/onflow/flow-go/module/util" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" +) + +const ( + // processTxErrorMessagesWorkersCount defines the number of workers that + // concurrently process transaction error messages in the job queue. + processTxErrorMessagesWorkersCount = 3 + + // defaultRetryDelay specifies the initial delay for the exponential backoff + // when the process of fetching transaction error messages fails. + // + // This delay increases with each retry attempt, up to the maximum defined by + // defaultMaxRetryDelay. + defaultRetryDelay = 1 * time.Second + + // defaultMaxRetryDelay specifies the maximum delay for the exponential backoff + // when the process of fetching transaction error messages fails. + // + // Once this delay is reached, the backoff will no longer increase with each retry. + defaultMaxRetryDelay = 5 * time.Minute +) + +// Engine represents the component responsible for managing and processing +// transaction result error messages. It retrieves, stores, +// and retries fetching of error messages from execution nodes, ensuring +// that they are processed and stored for sealed blocks. +// +// No errors are expected during normal operation. +type Engine struct { + *component.ComponentManager + + log zerolog.Logger + state protocol.State + headers storage.Headers + + // Job queue + txErrorMessagesConsumer *jobqueue.ComponentConsumer + // Notifiers for queue consumer + txErrorMessagesNotifier engine.Notifier + + txErrorMessagesCore *TxErrorMessagesCore // core logic for handling tx error messages +} + +// New creates a new Engine instance, initializing all necessary components +// for processing transaction result error messages. This includes setting +// up the job queue and the notifier for handling finalized blocks. +// +// No errors are expected during normal operation. +func New( + log zerolog.Logger, + state protocol.State, + headers storage.Headers, + txErrorMessagesProcessedHeight storage.ConsumerProgress, + txErrorMessagesCore *TxErrorMessagesCore, +) (*Engine, error) { + e := &Engine{ + log: log.With().Str("engine", "tx_error_messages_engine").Logger(), + state: state, + headers: headers, + txErrorMessagesCore: txErrorMessagesCore, + txErrorMessagesNotifier: engine.NewNotifier(), + } + + // jobqueue Jobs object that tracks sealed blocks by height. This is used by the txErrorMessagesConsumer + // to get a sequential list of sealed blocks. + sealedBlockReader := jobqueue.NewSealedBlockHeaderReader(state, headers) + + var err error + // Create a job queue that will process error messages for new sealed blocks. + // It listens to block finalization events from `txErrorMessagesNotifier`, then checks if there + // are new sealed blocks with `sealedBlockReader`. If there are, it starts workers to process + // them with `processTxResultErrorMessagesJob`, which fetches transaction error messages. At most + // `processTxErrorMessagesWorkersCount` workers will be created for concurrent processing. + // When a sealed block's error messages has been processed, it updates and persists the highest consecutive + // processed height with `txErrorMessagesProcessedHeight`. That way, if the node crashes, + // it reads the `txErrorMessagesProcessedHeight` and resume from `txErrorMessagesProcessedHeight + 1`. + // If the database is empty, rootHeight will be used to init the last processed height. + e.txErrorMessagesConsumer, err = jobqueue.NewComponentConsumer( + e.log.With().Str("engine", "tx_error_messages").Logger(), + e.txErrorMessagesNotifier.Channel(), + txErrorMessagesProcessedHeight, + sealedBlockReader, + e.state.Params().SealedRoot().Height, + e.processTxResultErrorMessagesJob, + processTxErrorMessagesWorkersCount, + 0, + ) + if err != nil { + return nil, fmt.Errorf("error creating transaction result error messages jobqueue: %w", err) + } + + // Add workers + e.ComponentManager = component.NewComponentManagerBuilder(). + AddWorker(e.runTxResultErrorMessagesConsumer). + Build() + + return e, nil +} + +// processTxResultErrorMessagesJob processes a job for transaction error messages by +// converting the job to a block and processing error messages. If processing +// fails for all attempts, it logs the error. +func (e *Engine) processTxResultErrorMessagesJob(ctx irrecoverable.SignalerContext, job module.Job, done func()) { + header, err := jobqueue.JobToBlockHeader(job) + if err != nil { + ctx.Throw(fmt.Errorf("failed to convert job to block: %w", err)) + } + + err = e.processErrorMessagesForBlock(ctx, header.ID()) + if err == nil { + done() + return + } + + e.log.Error(). + Err(err). + Str("job_id", string(job.ID())). + Msg("error encountered while processing transaction result error messages job") +} + +// runTxResultErrorMessagesConsumer runs the txErrorMessagesConsumer component +func (e *Engine) runTxResultErrorMessagesConsumer(ctx irrecoverable.SignalerContext, ready component.ReadyFunc) { + e.txErrorMessagesConsumer.Start(ctx) + + err := util.WaitClosed(ctx, e.txErrorMessagesConsumer.Ready()) + if err == nil { + ready() + } + + <-e.txErrorMessagesConsumer.Done() +} + +// OnFinalizedBlock is called by the follower engine after a block has been finalized and the state has been updated. +// Receives block finalized events from the finalization distributor and forwards them to the txErrorMessagesConsumer. +func (e *Engine) OnFinalizedBlock(*model.Block) { + e.txErrorMessagesNotifier.Notify() +} + +// processErrorMessagesForBlock processes transaction result error messages for block. +// If the process fails, it will retry, using exponential backoff. +// +// No errors are expected during normal operation. +func (e *Engine) processErrorMessagesForBlock(ctx context.Context, blockID flow.Identifier) error { + backoff := retry.NewExponential(defaultRetryDelay) + backoff = retry.WithCappedDuration(defaultMaxRetryDelay, backoff) + backoff = retry.WithJitterPercent(15, backoff) + + attempt := 0 + return retry.Do(ctx, backoff, func(context.Context) error { + if attempt > 0 { + e.log.Debug(). + Str("block_id", blockID.String()). + Uint64("attempt", uint64(attempt)). + Msgf("retrying process transaction result error messages") + + } + attempt++ + err := e.txErrorMessagesCore.HandleTransactionResultErrorMessages(ctx, blockID) + + return retry.RetryableError(err) + }) +} diff --git a/engine/access/ingestion/tx_error_messages/tx_error_messages_engine_test.go b/engine/access/ingestion/tx_error_messages/tx_error_messages_engine_test.go new file mode 100644 index 00000000000..44980d7c79f --- /dev/null +++ b/engine/access/ingestion/tx_error_messages/tx_error_messages_engine_test.go @@ -0,0 +1,249 @@ +package tx_error_messages + +import ( + "context" + "os" + "sync" + "testing" + "time" + + "github.com/dgraph-io/badger/v2" + execproto "github.com/onflow/flow/protobuf/go/flow/execution" + "github.com/rs/zerolog" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + hotmodel "github.com/onflow/flow-go/consensus/hotstuff/model" + accessmock "github.com/onflow/flow-go/engine/access/mock" + "github.com/onflow/flow-go/engine/access/rpc/backend" + connectionmock "github.com/onflow/flow-go/engine/access/rpc/connection/mock" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/irrecoverable" + protocol "github.com/onflow/flow-go/state/protocol/mock" + bstorage "github.com/onflow/flow-go/storage/badger" + storage "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/unittest" + "github.com/onflow/flow-go/utils/unittest/mocks" +) + +// TxErrorMessagesEngineSuite is a test suite for the transaction error messages engine. +// It sets up the necessary mocks and dependencies to test the functionality of +// handling transaction error messages. +type TxErrorMessagesEngineSuite struct { + suite.Suite + + log zerolog.Logger + proto struct { + state *protocol.FollowerState + snapshot *protocol.Snapshot + params *protocol.Params + } + headers *storage.Headers + receipts *storage.ExecutionReceipts + txErrorMessages *storage.TransactionResultErrorMessages + + enNodeIDs flow.IdentityList + execClient *accessmock.ExecutionAPIClient + connFactory *connectionmock.ConnectionFactory + + blockMap map[uint64]*flow.Block + rootBlock flow.Block + sealedBlock *flow.Header + + db *badger.DB + dbDir string + + ctx context.Context + cancel context.CancelFunc +} + +// TestTxErrorMessagesEngine runs the test suite for the transaction error messages engine. +func TestTxErrorMessagesEngine(t *testing.T) { + suite.Run(t, new(TxErrorMessagesEngineSuite)) +} + +// TearDownTest stops the engine and cleans up the db +func (s *TxErrorMessagesEngineSuite) TearDownTest() { + s.cancel() + err := os.RemoveAll(s.dbDir) + s.Require().NoError(err) +} + +func (s *TxErrorMessagesEngineSuite) SetupTest() { + s.log = zerolog.New(os.Stderr) + s.ctx, s.cancel = context.WithCancel(context.Background()) + s.db, s.dbDir = unittest.TempBadgerDB(s.T()) + // mock out protocol state + s.proto.state = protocol.NewFollowerState(s.T()) + s.proto.snapshot = protocol.NewSnapshot(s.T()) + s.proto.params = protocol.NewParams(s.T()) + s.execClient = accessmock.NewExecutionAPIClient(s.T()) + s.connFactory = connectionmock.NewConnectionFactory(s.T()) + s.headers = storage.NewHeaders(s.T()) + s.receipts = storage.NewExecutionReceipts(s.T()) + s.txErrorMessages = storage.NewTransactionResultErrorMessages(s.T()) + + blockCount := 5 + s.blockMap = make(map[uint64]*flow.Block, blockCount) + s.rootBlock = unittest.BlockFixture() + s.rootBlock.Header.Height = 0 + parent := s.rootBlock.Header + + for i := 0; i < blockCount; i++ { + block := unittest.BlockWithParentFixture(parent) + // update for next iteration + parent = block.Header + s.blockMap[block.Header.Height] = block + } + + s.sealedBlock = parent + + s.headers.On("ByHeight", mock.AnythingOfType("uint64")).Return( + mocks.ConvertStorageOutput( + mocks.StorageMapGetter(s.blockMap), + func(block *flow.Block) *flow.Header { return block.Header }, + ), + ).Maybe() + + s.proto.state.On("Params").Return(s.proto.params) + + // Mock the finalized and sealed root block header with height 0. + s.proto.params.On("FinalizedRoot").Return(s.rootBlock.Header, nil) + s.proto.params.On("SealedRoot").Return(s.rootBlock.Header, nil) + + s.proto.snapshot.On("Head").Return( + func() *flow.Header { + return s.sealedBlock + }, + nil, + ).Maybe() + + s.proto.state.On("Sealed").Return(s.proto.snapshot, nil) + s.proto.state.On("Final").Return(s.proto.snapshot, nil) + + // Create identities for 1 execution nodes. + s.enNodeIDs = unittest.IdentityListFixture(1, unittest.WithRole(flow.RoleExecution)) +} + +// initEngine creates a new instance of the transaction error messages engine +// and waits for it to start. It initializes the engine with mocked components and state. +func (s *TxErrorMessagesEngineSuite) initEngine(ctx irrecoverable.SignalerContext) *Engine { + processedTxErrorMessagesBlockHeight := bstorage.NewConsumerProgress( + s.db, + module.ConsumeProgressEngineTxErrorMessagesBlockHeight, + ) + + // Initialize the backend with the mocked state, blocks, headers, transactions, etc. + backend, err := backend.New(backend.Params{ + State: s.proto.state, + Headers: s.headers, + ExecutionReceipts: s.receipts, + ConnFactory: s.connFactory, + MaxHeightRange: backend.DefaultMaxHeightRange, + FixedExecutionNodeIDs: s.enNodeIDs.NodeIDs().Strings(), + Log: s.log, + SnapshotHistoryLimit: backend.DefaultSnapshotHistoryLimit, + Communicator: backend.NewNodeCommunicator(false), + ScriptExecutionMode: backend.IndexQueryModeExecutionNodesOnly, + TxResultQueryMode: backend.IndexQueryModeExecutionNodesOnly, + ChainID: flow.Testnet, + }) + require.NoError(s.T(), err) + + txResultErrorMessagesCore := NewTxErrorMessagesCore( + s.log, + s.proto.state, + backend, + s.receipts, + s.txErrorMessages, + s.enNodeIDs.NodeIDs(), + nil, + ) + + eng, err := New( + s.log, + s.proto.state, + s.headers, + processedTxErrorMessagesBlockHeight, + txResultErrorMessagesCore, + ) + require.NoError(s.T(), err) + + eng.ComponentManager.Start(ctx) + <-eng.Ready() + + return eng +} + +// TestOnFinalizedBlockHandleTxErrorMessages tests the handling of transaction error messages +// when a new finalized block is processed. It verifies that the engine fetches transaction +// error messages from execution nodes and stores them in the database. +func (s *TxErrorMessagesEngineSuite) TestOnFinalizedBlockHandleTxErrorMessages() { + irrecoverableCtx := irrecoverable.NewMockSignalerContext(s.T(), s.ctx) + + block := unittest.BlockWithParentFixture(s.sealedBlock) + + s.blockMap[block.Header.Height] = block + s.sealedBlock = block.Header + + hotstuffBlock := hotmodel.Block{ + BlockID: block.ID(), + } + + // mock the connection factory + s.connFactory.On("GetExecutionAPIClient", mock.Anything).Return(s.execClient, &mockCloser{}, nil) + + s.proto.snapshot.On("Identities", mock.Anything).Return(s.enNodeIDs, nil) + s.proto.state.On("AtBlockID", mock.Anything).Return(s.proto.snapshot) + + count := 6 + wg := sync.WaitGroup{} + wg.Add(count) + + for _, b := range s.blockMap { + blockID := b.ID() + + // Mock the protocol snapshot to return fixed execution node IDs. + setupReceiptsForBlock(s.receipts, b, s.enNodeIDs.NodeIDs()[0]) + + // Mock the txErrorMessages storage to confirm that error messages do not exist yet. + s.txErrorMessages.On("Exists", blockID). + Return(false, nil).Once() + + // Create mock transaction results with a mix of failed and non-failed transactions. + resultsByBlockID := mockTransactionResultsByBlock(5) + + // Prepare a request to fetch transaction error messages by block ID from execution nodes. + exeEventReq := &execproto.GetTransactionErrorMessagesByBlockIDRequest{ + BlockId: blockID[:], + } + + s.execClient.On("GetTransactionErrorMessagesByBlockID", mock.Anything, exeEventReq). + Return(createTransactionErrorMessagesResponse(resultsByBlockID), nil).Once() + + // Prepare the expected transaction error messages that should be stored. + expectedStoreTxErrorMessages := createExpectedTxErrorMessages(resultsByBlockID, s.enNodeIDs.NodeIDs()[0]) + + // Mock the storage of the fetched error messages into the protocol database. + s.txErrorMessages.On("Store", blockID, expectedStoreTxErrorMessages).Return(nil). + Run(func(args mock.Arguments) { + // Ensure the test does not complete its work faster than necessary + wg.Done() + }).Once() + } + + eng := s.initEngine(irrecoverableCtx) + // process the block through the finalized callback + eng.OnFinalizedBlock(&hotstuffBlock) + + // Verify that all transaction error messages were processed within the timeout. + unittest.RequireReturnsBefore(s.T(), wg.Wait, 2*time.Second, "expect to process new block before timeout") + + // Ensure all expectations were met. + s.txErrorMessages.AssertExpectations(s.T()) + s.headers.AssertExpectations(s.T()) + s.proto.state.AssertExpectations(s.T()) + s.execClient.AssertExpectations(s.T()) +} diff --git a/engine/access/rpc/backend/backend.go b/engine/access/rpc/backend/backend.go index 59254e617bd..a48f335670d 100644 --- a/engine/access/rpc/backend/backend.go +++ b/engine/access/rpc/backend/backend.go @@ -17,10 +17,10 @@ import ( "github.com/onflow/flow-go/engine/access/rpc/connection" "github.com/onflow/flow-go/engine/access/subscription" "github.com/onflow/flow-go/engine/common/rpc" + commonrpc "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/version" "github.com/onflow/flow-go/fvm/blueprints" "github.com/onflow/flow-go/model/flow" - "github.com/onflow/flow-go/model/flow/filter" "github.com/onflow/flow-go/module" "github.com/onflow/flow-go/module/counters" "github.com/onflow/flow-go/module/execution" @@ -29,12 +29,6 @@ import ( "github.com/onflow/flow-go/storage" ) -// minExecutionNodesCnt is the minimum number of execution nodes expected to have sent the execution receipt for a block -const minExecutionNodesCnt = 2 - -// maxAttemptsForExecutionReceipt is the maximum number of attempts to find execution receipts for a given block ID -const maxAttemptsForExecutionReceipt = 3 - // DefaultMaxHeightRange is the default maximum size of range requests. const DefaultMaxHeightRange = 250 @@ -99,6 +93,7 @@ type Params struct { Transactions storage.Transactions ExecutionReceipts storage.ExecutionReceipts ExecutionResults storage.ExecutionResults + TxResultErrorMessages storage.TransactionResultErrorMessages ChainID flow.ChainID AccessMetrics module.AccessMetrics ConnFactory connection.ConnectionFactory @@ -110,7 +105,6 @@ type Params struct { SnapshotHistoryLimit int Communicator Communicator TxResultCacheSize uint - TxErrorMessagesCacheSize uint ScriptExecutor execution.ScriptExecutor ScriptExecutionMode IndexQueryMode CheckPayerBalanceMode access.PayerBalanceMode @@ -148,18 +142,6 @@ func New(params Params) (*Backend, error) { } } - // NOTE: The transaction error message cache is currently only used by the access node and not by the observer node. - // To avoid introducing unnecessary command line arguments in the observer, one case could be that the error - // message cache is nil for the observer node. - var txErrorMessagesCache *lru.Cache[flow.Identifier, string] - - if params.TxErrorMessagesCacheSize > 0 { - txErrorMessagesCache, err = lru.New[flow.Identifier, string](int(params.TxErrorMessagesCacheSize)) - if err != nil { - return nil, fmt.Errorf("failed to init cache for transaction error messages: %w", err) - } - } - // the system tx is hardcoded and never changes during runtime systemTx, err := blueprints.SystemChunkTransaction(params.ChainID.Chain()) if err != nil { @@ -261,6 +243,7 @@ func New(params Params) (*Backend, error) { chainID: params.ChainID, transactions: params.Transactions, executionReceipts: params.ExecutionReceipts, + txResultErrorMessages: params.TxResultErrorMessages, transactionValidator: txValidator, transactionMetrics: params.AccessMetrics, retry: retry, @@ -268,7 +251,6 @@ func New(params Params) (*Backend, error) { previousAccessNodes: params.HistoricalAccessNodes, nodeCommunicator: params.Communicator, txResultCache: txResCache, - txErrorMessagesCache: txErrorMessagesCache, txResultQueryMode: params.TxResultQueryMode, systemTx: systemTx, systemTxID: systemTxID, @@ -288,12 +270,12 @@ func New(params Params) (*Backend, error) { retry.SetBackend(b) - preferredENIdentifiers, err = identifierList(params.PreferredExecutionNodeIDs) + preferredENIdentifiers, err = commonrpc.IdentifierList(params.PreferredExecutionNodeIDs) if err != nil { return nil, fmt.Errorf("failed to convert node id string to Flow Identifier for preferred EN map: %w", err) } - fixedENIdentifiers, err = identifierList(params.FixedExecutionNodeIDs) + fixedENIdentifiers, err = commonrpc.IdentifierList(params.FixedExecutionNodeIDs) if err != nil { return nil, fmt.Errorf("failed to convert node id string to Flow Identifier for fixed EN map: %w", err) } @@ -301,18 +283,6 @@ func New(params Params) (*Backend, error) { return b, nil } -func identifierList(ids []string) (flow.IdentifierList, error) { - idList := make(flow.IdentifierList, len(ids)) - for i, idStr := range ids { - id, err := flow.HexStringToIdentifier(idStr) - if err != nil { - return nil, fmt.Errorf("failed to convert node id string %s to Flow Identifier: %w", id, err) - } - idList[i] = id - } - return idList, nil -} - func configureTransactionValidator( state protocol.State, chainID flow.ChainID, @@ -420,229 +390,6 @@ func (b *Backend) GetNetworkParameters(_ context.Context) access.NetworkParamete } } -// executionNodesForBlockID returns upto maxNodesCnt number of randomly chosen execution node identities -// which have executed the given block ID. -// If no such execution node is found, an InsufficientExecutionReceipts error is returned. -func executionNodesForBlockID( - ctx context.Context, - blockID flow.Identifier, - executionReceipts storage.ExecutionReceipts, - state protocol.State, - log zerolog.Logger, -) (flow.IdentitySkeletonList, error) { - var ( - executorIDs flow.IdentifierList - err error - ) - - // check if the block ID is of the root block. If it is then don't look for execution receipts since they - // will not be present for the root block. - rootBlock := state.Params().FinalizedRoot() - - if rootBlock.ID() == blockID { - executorIdentities, err := state.Final().Identities(filter.HasRole[flow.Identity](flow.RoleExecution)) - if err != nil { - return nil, fmt.Errorf("failed to retreive execution IDs for block ID %v: %w", blockID, err) - } - executorIDs = executorIdentities.NodeIDs() - } else { - // try to find at least minExecutionNodesCnt execution node ids from the execution receipts for the given blockID - for attempt := 0; attempt < maxAttemptsForExecutionReceipt; attempt++ { - executorIDs, err = findAllExecutionNodes(blockID, executionReceipts, log) - if err != nil { - return nil, err - } - - if len(executorIDs) >= minExecutionNodesCnt { - break - } - - // log the attempt - log.Debug().Int("attempt", attempt).Int("max_attempt", maxAttemptsForExecutionReceipt). - Int("execution_receipts_found", len(executorIDs)). - Str("block_id", blockID.String()). - Msg("insufficient execution receipts") - - // if one or less execution receipts may have been received then re-query - // in the hope that more might have been received by now - - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(100 * time.Millisecond << time.Duration(attempt)): - // retry after an exponential backoff - } - } - - receiptCnt := len(executorIDs) - // if less than minExecutionNodesCnt execution receipts have been received so far, then return random ENs - if receiptCnt < minExecutionNodesCnt { - newExecutorIDs, err := state.AtBlockID(blockID).Identities(filter.HasRole[flow.Identity](flow.RoleExecution)) - if err != nil { - return nil, fmt.Errorf("failed to retreive execution IDs for block ID %v: %w", blockID, err) - } - executorIDs = newExecutorIDs.NodeIDs() - } - } - - // choose from the preferred or fixed execution nodes - subsetENs, err := chooseExecutionNodes(state, executorIDs) - if err != nil { - return nil, fmt.Errorf("failed to retreive execution IDs for block ID %v: %w", blockID, err) - } - - if len(subsetENs) == 0 { - return nil, fmt.Errorf("no matching execution node found for block ID %v", blockID) - } - - return subsetENs, nil -} - -// findAllExecutionNodes find all the execution nodes ids from the execution receipts that have been received for the -// given blockID -func findAllExecutionNodes( - blockID flow.Identifier, - executionReceipts storage.ExecutionReceipts, - log zerolog.Logger, -) (flow.IdentifierList, error) { - // lookup the receipt's storage with the block ID - allReceipts, err := executionReceipts.ByBlockID(blockID) - if err != nil { - return nil, fmt.Errorf("failed to retreive execution receipts for block ID %v: %w", blockID, err) - } - - executionResultMetaList := make(flow.ExecutionReceiptMetaList, 0, len(allReceipts)) - for _, r := range allReceipts { - executionResultMetaList = append(executionResultMetaList, r.Meta()) - } - executionResultGroupedMetaList := executionResultMetaList.GroupByResultID() - - // maximum number of matching receipts found so far for any execution result id - maxMatchedReceiptCnt := 0 - // execution result id key for the highest number of matching receipts in the identicalReceipts map - var maxMatchedReceiptResultID flow.Identifier - - // find the largest list of receipts which have the same result ID - for resultID, executionReceiptList := range executionResultGroupedMetaList { - currentMatchedReceiptCnt := executionReceiptList.Size() - if currentMatchedReceiptCnt > maxMatchedReceiptCnt { - maxMatchedReceiptCnt = currentMatchedReceiptCnt - maxMatchedReceiptResultID = resultID - } - } - - // if there are more than one execution result for the same block ID, log as error - if executionResultGroupedMetaList.NumberGroups() > 1 { - identicalReceiptsStr := fmt.Sprintf("%v", flow.GetIDs(allReceipts)) - log.Error(). - Str("block_id", blockID.String()). - Str("execution_receipts", identicalReceiptsStr). - Msg("execution receipt mismatch") - } - - // pick the largest list of matching receipts - matchingReceiptMetaList := executionResultGroupedMetaList.GetGroup(maxMatchedReceiptResultID) - - metaReceiptGroupedByExecutorID := matchingReceiptMetaList.GroupByExecutorID() - - // collect all unique execution node ids from the receipts - var executorIDs flow.IdentifierList - for executorID := range metaReceiptGroupedByExecutorID { - executorIDs = append(executorIDs, executorID) - } - - return executorIDs, nil -} - -// chooseExecutionNodes finds the subset of execution nodes defined in the identity table by first -// choosing the preferred execution nodes which have executed the transaction. If no such preferred -// execution nodes are found, then the fixed execution nodes defined in the identity table are returned -// If neither preferred nor fixed nodes are defined, then all execution node matching the executor IDs are returned. -// e.g. If execution nodes in identity table are {1,2,3,4}, preferred ENs are defined as {2,3,4} -// and the executor IDs is {1,2,3}, then {2, 3} is returned as the chosen subset of ENs -func chooseExecutionNodes(state protocol.State, executorIDs flow.IdentifierList) (flow.IdentitySkeletonList, error) { - allENs, err := state.Final().Identities(filter.HasRole[flow.Identity](flow.RoleExecution)) - if err != nil { - return nil, fmt.Errorf("failed to retrieve all execution IDs: %w", err) - } - - // choose from preferred EN IDs - if len(preferredENIdentifiers) > 0 { - chosenIDs := chooseFromPreferredENIDs(allENs, executorIDs) - return chosenIDs.ToSkeleton(), nil - } - - // if no preferred EN ID is found, then choose from the fixed EN IDs - if len(fixedENIdentifiers) > 0 { - // choose fixed ENs which have executed the transaction - chosenIDs := allENs.Filter(filter.And( - filter.HasNodeID[flow.Identity](fixedENIdentifiers...), - filter.HasNodeID[flow.Identity](executorIDs...), - )) - if len(chosenIDs) > 0 { - return chosenIDs.ToSkeleton(), nil - } - // if no such ENs are found, then just choose all fixed ENs - chosenIDs = allENs.Filter(filter.HasNodeID[flow.Identity](fixedENIdentifiers...)) - return chosenIDs.ToSkeleton(), nil - } - - // if no preferred or fixed ENs have been specified, then return all executor IDs i.e., no preference at all - return allENs.Filter(filter.HasNodeID[flow.Identity](executorIDs...)).ToSkeleton(), nil -} - -// chooseFromPreferredENIDs finds the subset of execution nodes if preferred execution nodes are defined. -// If preferredENIdentifiers is set and there are less than maxNodesCnt nodes selected, than the list is padded up to -// maxNodesCnt nodes using the following order: -// 1. Use any EN with a receipt. -// 2. Use any preferred node not already selected. -// 3. Use any EN not already selected. -func chooseFromPreferredENIDs(allENs flow.IdentityList, executorIDs flow.IdentifierList) flow.IdentityList { - var chosenIDs flow.IdentityList - - // filter for both preferred and executor IDs - chosenIDs = allENs.Filter(filter.And( - filter.HasNodeID[flow.Identity](preferredENIdentifiers...), - filter.HasNodeID[flow.Identity](executorIDs...), - )) - - if len(chosenIDs) >= maxNodesCnt { - return chosenIDs - } - - // function to add nodes to chosenIDs if they are not already included - addIfNotExists := func(candidates flow.IdentityList) { - for _, en := range candidates { - _, exists := chosenIDs.ByNodeID(en.NodeID) - if !exists { - chosenIDs = append(chosenIDs, en) - if len(chosenIDs) >= maxNodesCnt { - return - } - } - } - } - - // add any EN with a receipt - receiptENs := allENs.Filter(filter.HasNodeID[flow.Identity](executorIDs...)) - addIfNotExists(receiptENs) - if len(chosenIDs) >= maxNodesCnt { - return chosenIDs - } - - // add any preferred node not already selected - preferredENs := allENs.Filter(filter.HasNodeID[flow.Identity](preferredENIdentifiers...)) - addIfNotExists(preferredENs) - if len(chosenIDs) >= maxNodesCnt { - return chosenIDs - } - - // add any EN not already selected - addIfNotExists(allENs) - - return chosenIDs -} - // resolveHeightError processes errors returned during height-based queries. // If the error is due to a block not being found, this function determines whether the queried // height falls outside the node's accessible range and provides context-sensitive error messages diff --git a/engine/access/rpc/backend/backend_accounts.go b/engine/access/rpc/backend/backend_accounts.go index 252a834364a..f3a38219a31 100644 --- a/engine/access/rpc/backend/backend_accounts.go +++ b/engine/access/rpc/backend/backend_accounts.go @@ -14,6 +14,7 @@ import ( "github.com/onflow/flow-go/engine/access/rpc/connection" "github.com/onflow/flow-go/engine/common/rpc" + commonrpc "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/rpc/convert" fvmerrors "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/model/flow" @@ -419,7 +420,15 @@ func (b *backendAccounts) getAccountFromAnyExeNode( BlockId: blockID[:], } - execNodes, err := executionNodesForBlockID(ctx, blockID, b.executionReceipts, b.state, b.log) + execNodes, err := commonrpc.ExecutionNodesForBlockID( + ctx, + blockID, + b.executionReceipts, + b.state, + b.log, + preferredENIdentifiers, + fixedENIdentifiers, + ) if err != nil { return nil, rpc.ConvertError(err, "failed to find execution node to query", codes.Internal) } diff --git a/engine/access/rpc/backend/backend_events.go b/engine/access/rpc/backend/backend_events.go index b808e20c176..9f762672996 100644 --- a/engine/access/rpc/backend/backend_events.go +++ b/engine/access/rpc/backend/backend_events.go @@ -303,7 +303,13 @@ func (b *backendEvents) getBlockEventsFromExecutionNode( // choose the last block ID to find the list of execution nodes lastBlockID := blockIDs[len(blockIDs)-1] - execNodes, err := executionNodesForBlockID(ctx, lastBlockID, b.executionReceipts, b.state, b.log) + execNodes, err := rpc.ExecutionNodesForBlockID(ctx, + lastBlockID, + b.executionReceipts, + b.state, + b.log, + preferredENIdentifiers, + fixedENIdentifiers) if err != nil { return nil, rpc.ConvertError(err, "failed to retrieve events from execution node", codes.Internal) } diff --git a/engine/access/rpc/backend/backend_scripts.go b/engine/access/rpc/backend/backend_scripts.go index 0efb21839b5..7d22f117912 100644 --- a/engine/access/rpc/backend/backend_scripts.go +++ b/engine/access/rpc/backend/backend_scripts.go @@ -13,6 +13,7 @@ import ( "github.com/onflow/flow-go/engine/access/rpc/connection" "github.com/onflow/flow-go/engine/common/rpc" + commonrpc "github.com/onflow/flow-go/engine/common/rpc" fvmerrors "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" @@ -224,7 +225,7 @@ func (b *backendScripts) executeScriptOnAvailableExecutionNodes( r *scriptExecutionRequest, ) ([]byte, time.Duration, error) { // find few execution nodes which have executed the block earlier and provided an execution receipt for it - executors, err := executionNodesForBlockID(ctx, r.blockID, b.executionReceipts, b.state, b.log) + executors, err := commonrpc.ExecutionNodesForBlockID(ctx, r.blockID, b.executionReceipts, b.state, b.log, preferredENIdentifiers, fixedENIdentifiers) if err != nil { return nil, 0, status.Errorf(codes.Internal, "failed to find script executors at blockId %v: %v", r.blockID.String(), err) } diff --git a/engine/access/rpc/backend/backend_stream_blocks_test.go b/engine/access/rpc/backend/backend_stream_blocks_test.go index 69aa9e67823..6e655c8afda 100644 --- a/engine/access/rpc/backend/backend_stream_blocks_test.go +++ b/engine/access/rpc/backend/backend_stream_blocks_test.go @@ -148,15 +148,14 @@ func (s *BackendBlocksSuite) SetupTest() { // backendParams returns the Params configuration for the backend. func (s *BackendBlocksSuite) backendParams() Params { return Params{ - State: s.state, - Blocks: s.blocks, - Headers: s.headers, - ChainID: s.chainID, - MaxHeightRange: DefaultMaxHeightRange, - SnapshotHistoryLimit: DefaultSnapshotHistoryLimit, - AccessMetrics: metrics.NewNoopCollector(), - Log: s.log, - TxErrorMessagesCacheSize: 1000, + State: s.state, + Blocks: s.blocks, + Headers: s.headers, + ChainID: s.chainID, + MaxHeightRange: DefaultMaxHeightRange, + SnapshotHistoryLimit: DefaultSnapshotHistoryLimit, + AccessMetrics: metrics.NewNoopCollector(), + Log: s.log, SubscriptionHandler: subscription.NewSubscriptionHandler( s.log, s.broadcaster, diff --git a/engine/access/rpc/backend/backend_stream_transactions_test.go b/engine/access/rpc/backend/backend_stream_transactions_test.go index ec8ff353bf3..24cdf601f17 100644 --- a/engine/access/rpc/backend/backend_stream_transactions_test.go +++ b/engine/access/rpc/backend/backend_stream_transactions_test.go @@ -192,22 +192,21 @@ func (s *TransactionStatusSuite) TearDownTest() { // backendParams returns the Params configuration for the backend. func (s *TransactionStatusSuite) backendParams() Params { return Params{ - State: s.state, - Blocks: s.blocks, - Headers: s.headers, - Collections: s.collections, - Transactions: s.transactions, - ExecutionReceipts: s.receipts, - ExecutionResults: s.results, - ChainID: s.chainID, - CollectionRPC: s.colClient, - MaxHeightRange: DefaultMaxHeightRange, - SnapshotHistoryLimit: DefaultSnapshotHistoryLimit, - Communicator: NewNodeCommunicator(false), - AccessMetrics: metrics.NewNoopCollector(), - Log: s.log, - TxErrorMessagesCacheSize: 1000, - BlockTracker: s.blockTracker, + State: s.state, + Blocks: s.blocks, + Headers: s.headers, + Collections: s.collections, + Transactions: s.transactions, + ExecutionReceipts: s.receipts, + ExecutionResults: s.results, + ChainID: s.chainID, + CollectionRPC: s.colClient, + MaxHeightRange: DefaultMaxHeightRange, + SnapshotHistoryLimit: DefaultSnapshotHistoryLimit, + Communicator: NewNodeCommunicator(false), + AccessMetrics: metrics.NewNoopCollector(), + Log: s.log, + BlockTracker: s.blockTracker, SubscriptionHandler: subscription.NewSubscriptionHandler( s.log, s.broadcaster, diff --git a/engine/access/rpc/backend/backend_test.go b/engine/access/rpc/backend/backend_test.go index 73edb75791e..617135ad1aa 100644 --- a/engine/access/rpc/backend/backend_test.go +++ b/engine/access/rpc/backend/backend_test.go @@ -29,6 +29,7 @@ import ( backendmock "github.com/onflow/flow-go/engine/access/rpc/backend/mock" "github.com/onflow/flow-go/engine/access/rpc/connection" connectionmock "github.com/onflow/flow-go/engine/access/rpc/connection/mock" + commonrpc "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/engine/common/version" "github.com/onflow/flow-go/fvm/blueprints" @@ -72,6 +73,7 @@ type Suite struct { results *storagemock.ExecutionResults transactionResults *storagemock.LightTransactionResults events *storagemock.Events + txErrorMessages *storagemock.TransactionResultErrorMessages db *badger.DB dbDir string @@ -112,6 +114,7 @@ func (suite *Suite) SetupTest() { suite.collections = new(storagemock.Collections) suite.receipts = new(storagemock.ExecutionReceipts) suite.results = new(storagemock.ExecutionResults) + suite.txErrorMessages = storagemock.NewTransactionResultErrorMessages(suite.T()) suite.colClient = new(accessmock.AccessAPIClient) suite.execClient = new(accessmock.ExecutionAPIClient) suite.transactionResults = storagemock.NewLightTransactionResults(suite.T()) @@ -909,10 +912,6 @@ func (suite *Suite) TestGetTransactionResultByIndex() { suite.state.On("Final").Return(suite.snapshot, nil).Maybe() suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) - // create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) - exeEventReq := &execproto.GetTransactionByIndexRequest{ BlockId: blockId[:], Index: index, @@ -924,7 +923,7 @@ func (suite *Suite) TestGetTransactionResultByIndex() { params := suite.defaultBackendParams() // the connection factory should be used to get the execution node client - params.ConnFactory = connFactory + params.ConnFactory = suite.setupConnectionFactory() params.FixedExecutionNodeIDs = (fixedENIDs.NodeIDs()).Strings() backend, err := New(params) @@ -979,10 +978,6 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { suite.state.On("Final").Return(suite.snapshot, nil).Maybe() suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) - // create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) - exeEventReq := &execproto.GetTransactionsByBlockIDRequest{ BlockId: blockId[:], } @@ -992,7 +987,7 @@ func (suite *Suite) TestGetTransactionResultsByBlockID() { } // the connection factory should be used to get the execution node client - params.ConnFactory = connFactory + params.ConnFactory = suite.setupConnectionFactory() params.FixedExecutionNodeIDs = (fixedENIDs.NodeIDs()).Strings() backend, err := New(params) @@ -1072,10 +1067,6 @@ func (suite *Suite) TestTransactionStatusTransition() { suite.state.On("Final").Return(suite.snapshot, nil).Maybe() suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) - // create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) - exeEventReq := &execproto.GetTransactionResultRequest{ BlockId: blockID[:], TransactionId: txID[:], @@ -1087,7 +1078,7 @@ func (suite *Suite) TestTransactionStatusTransition() { params := suite.defaultBackendParams() // the connection factory should be used to get the execution node client - params.ConnFactory = connFactory + params.ConnFactory = suite.setupConnectionFactory() params.FixedExecutionNodeIDs = (fixedENIDs.NodeIDs()).Strings() backend, err := New(params) @@ -1730,7 +1721,7 @@ func (suite *Suite) TestGetNetworkParameters() { suite.Require().Equal(expectedChainID, actual.ChainID) } -// TestExecutionNodesForBlockID tests the common method backend.executionNodesForBlockID used for serving all API calls +// TestExecutionNodesForBlockID tests the common method backend.ExecutionNodesForBlockID used for serving all API calls // that need to talk to an execution node. func (suite *Suite) TestExecutionNodesForBlockID() { @@ -1796,7 +1787,7 @@ func (suite *Suite) TestExecutionNodesForBlockID() { expectedENs = flow.IdentityList{} } - allExecNodes, err := executionNodesForBlockID(context.Background(), block.ID(), suite.receipts, suite.state, suite.log) + allExecNodes, err := commonrpc.ExecutionNodesForBlockID(context.Background(), block.ID(), suite.receipts, suite.state, suite.log, preferredENIdentifiers, fixedENIdentifiers) require.NoError(suite.T(), err) execNodeSelectorFactory := NodeSelectorFactory{circuitBreakerEnabled: false} @@ -1810,7 +1801,7 @@ func (suite *Suite) TestExecutionNodesForBlockID() { { expectedENs := expectedENs.ToSkeleton() - if len(expectedENs) > maxNodesCnt { + if len(expectedENs) > commonrpc.MaxNodesCnt { for _, actual := range actualList { require.Contains(suite.T(), expectedENs, actual) } @@ -1819,7 +1810,7 @@ func (suite *Suite) TestExecutionNodesForBlockID() { } } } - // if we don't find sufficient receipts, executionNodesForBlockID should return a list of random ENs + // if we don't find sufficient receipts, ExecutionNodesForBlockID should return a list of random ENs suite.Run("insufficient receipts return random ENs in State", func() { // return no receipts at all attempts attempt1Receipts = flow.ExecutionReceiptList{} @@ -1827,7 +1818,8 @@ func (suite *Suite) TestExecutionNodesForBlockID() { attempt3Receipts = flow.ExecutionReceiptList{} suite.state.On("AtBlockID", mock.Anything).Return(suite.snapshot) - allExecNodes, err := executionNodesForBlockID(context.Background(), block.ID(), suite.receipts, suite.state, suite.log) + allExecNodes, err := commonrpc.ExecutionNodesForBlockID(context.Background(), block.ID(), suite.receipts, suite.state, suite.log, preferredENIdentifiers, + fixedENIdentifiers) require.NoError(suite.T(), err) execNodeSelectorFactory := NodeSelectorFactory{circuitBreakerEnabled: false} @@ -1839,7 +1831,7 @@ func (suite *Suite) TestExecutionNodesForBlockID() { actualList = append(actualList, actual) } - require.Equal(suite.T(), len(actualList), maxNodesCnt) + require.Equal(suite.T(), len(actualList), commonrpc.MaxNodesCnt) }) // if no preferred or fixed ENs are specified, the ExecutionNodesForBlockID function should @@ -1860,7 +1852,7 @@ func (suite *Suite) TestExecutionNodesForBlockID() { suite.Run("two preferred ENs with zero fixed EN", func() { // mark the first two ENs as preferred preferredENs := allExecutionNodes[0:2] - expectedList := allExecutionNodes[0:maxNodesCnt] + expectedList := allExecutionNodes[0:commonrpc.MaxNodesCnt] testExecutionNodesForBlockID(preferredENs, nil, expectedList) }) // if both are specified, the ExecutionNodesForBlockID function should @@ -1870,7 +1862,7 @@ func (suite *Suite) TestExecutionNodesForBlockID() { fixedENs := allExecutionNodes[0:5] // mark the first two of the fixed ENs as preferred ENs preferredENs := fixedENs[0:2] - expectedList := fixedENs[0:maxNodesCnt] + expectedList := fixedENs[0:commonrpc.MaxNodesCnt] testExecutionNodesForBlockID(preferredENs, fixedENs, expectedList) }) // if both are specified, but the preferred ENs don't match the ExecutorIDs in the ER, @@ -1895,7 +1887,7 @@ func (suite *Suite) TestExecutionNodesForBlockID() { currentAttempt = 0 // mark the first two ENs as preferred preferredENs := allExecutionNodes[0:2] - expectedList := allExecutionNodes[0:maxNodesCnt] + expectedList := allExecutionNodes[0:commonrpc.MaxNodesCnt] testExecutionNodesForBlockID(preferredENs, nil, expectedList) }) // if preferredENIdentifiers was set and there are less than maxNodesCnt nodes selected than check the order @@ -1916,10 +1908,10 @@ func (suite *Suite) TestExecutionNodesForBlockID() { additionalNode[0], } - chosenIDs := chooseFromPreferredENIDs(allExecutionNodes, executorIDs) + chosenIDs := commonrpc.ChooseFromPreferredENIDs(allExecutionNodes, executorIDs, preferredENIdentifiers) require.ElementsMatch(suite.T(), chosenIDs, expectedOrder) - require.Equal(suite.T(), len(chosenIDs), maxNodesCnt) + require.Equal(suite.T(), len(chosenIDs), commonrpc.MaxNodesCnt) }) } @@ -1965,13 +1957,9 @@ func (suite *Suite) TestGetTransactionResultEventEncodingVersion() { suite.state.On("Final").Return(suite.snapshot, nil).Maybe() suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) - // create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) - params := suite.defaultBackendParams() // the connection factory should be used to get the execution node client - params.ConnFactory = connFactory + params.ConnFactory = suite.setupConnectionFactory() params.FixedExecutionNodeIDs = (fixedENIDs.NodeIDs()).Strings() backend, err := New(params) @@ -2031,13 +2019,9 @@ func (suite *Suite) TestGetTransactionResultByIndexAndBlockIdEventEncodingVersio suite.state.On("Final").Return(suite.snapshot, nil).Maybe() suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) - // create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) - params := suite.defaultBackendParams() // the connection factory should be used to get the execution node client - params.ConnFactory = connFactory + params.ConnFactory = suite.setupConnectionFactory() params.FixedExecutionNodeIDs = (fixedENIDs.NodeIDs()).Strings() backend, err := New(params) @@ -2131,17 +2115,13 @@ func (suite *Suite) TestNodeCommunicator() { suite.state.On("Final").Return(suite.snapshot, nil).Maybe() suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) - // create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) - exeEventReq := &execproto.GetTransactionsByBlockIDRequest{ BlockId: blockId[:], } params := suite.defaultBackendParams() // the connection factory should be used to get the execution node client - params.ConnFactory = connFactory + params.ConnFactory = suite.setupConnectionFactory() params.FixedExecutionNodeIDs = (fixedENIDs.NodeIDs()).Strings() // Left only one preferred execution node params.PreferredExecutionNodeIDs = []string{fixedENIDs[0].NodeID.String()} @@ -2219,25 +2199,24 @@ func generateEncodedEvents(t *testing.T, n int) ([]flow.Event, []flow.Event) { func (suite *Suite) defaultBackendParams() Params { return Params{ - State: suite.state, - Blocks: suite.blocks, - Headers: suite.headers, - Collections: suite.collections, - Transactions: suite.transactions, - ExecutionReceipts: suite.receipts, - ExecutionResults: suite.results, - ChainID: suite.chainID, - CollectionRPC: suite.colClient, - MaxHeightRange: DefaultMaxHeightRange, - SnapshotHistoryLimit: DefaultSnapshotHistoryLimit, - Communicator: NewNodeCommunicator(false), - AccessMetrics: metrics.NewNoopCollector(), - Log: suite.log, - TxErrorMessagesCacheSize: 1000, - BlockTracker: nil, - TxResultQueryMode: IndexQueryModeExecutionNodesOnly, - LastFullBlockHeight: suite.lastFullBlockHeight, - VersionControl: suite.versionControl, + State: suite.state, + Blocks: suite.blocks, + Headers: suite.headers, + Collections: suite.collections, + Transactions: suite.transactions, + ExecutionReceipts: suite.receipts, + ExecutionResults: suite.results, + ChainID: suite.chainID, + CollectionRPC: suite.colClient, + MaxHeightRange: DefaultMaxHeightRange, + SnapshotHistoryLimit: DefaultSnapshotHistoryLimit, + Communicator: NewNodeCommunicator(false), + AccessMetrics: metrics.NewNoopCollector(), + Log: suite.log, + BlockTracker: nil, + TxResultQueryMode: IndexQueryModeExecutionNodesOnly, + LastFullBlockHeight: suite.lastFullBlockHeight, + VersionControl: suite.versionControl, } } diff --git a/engine/access/rpc/backend/backend_transactions.go b/engine/access/rpc/backend/backend_transactions.go index 7436863f9af..e7596fd2f65 100644 --- a/engine/access/rpc/backend/backend_transactions.go +++ b/engine/access/rpc/backend/backend_transactions.go @@ -17,6 +17,7 @@ import ( "github.com/onflow/flow-go/access" "github.com/onflow/flow-go/engine/access/rpc/connection" "github.com/onflow/flow-go/engine/common/rpc" + commonrpc "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" @@ -25,23 +26,28 @@ import ( "github.com/onflow/flow-go/storage" ) +const DefaultFailedErrorMessage = "failed" + type backendTransactions struct { *TransactionsLocalDataProvider - staticCollectionRPC accessproto.AccessAPIClient // rpc client tied to a fixed collection node - transactions storage.Transactions - executionReceipts storage.ExecutionReceipts - chainID flow.ChainID - transactionMetrics module.TransactionMetrics - transactionValidator *access.TransactionValidator - retry *Retry - connFactory connection.ConnectionFactory - - previousAccessNodes []accessproto.AccessAPIClient - log zerolog.Logger - nodeCommunicator Communicator - txResultCache *lru.Cache[flow.Identifier, *access.TransactionResult] - txErrorMessagesCache *lru.Cache[flow.Identifier, string] // cache for transactions error messages, indexed by hash(block_id, tx_id). - txResultQueryMode IndexQueryMode + staticCollectionRPC accessproto.AccessAPIClient // rpc client tied to a fixed collection node + transactions storage.Transactions + executionReceipts storage.ExecutionReceipts + // NOTE: The transaction error message is currently only used by the access node and not by the observer node. + // To avoid introducing unnecessary command line arguments in the observer, one case could be that the error + // message cache is nil for the observer node. + txResultErrorMessages storage.TransactionResultErrorMessages + chainID flow.ChainID + transactionMetrics module.TransactionMetrics + transactionValidator *access.TransactionValidator + retry *Retry + connFactory connection.ConnectionFactory + + previousAccessNodes []accessproto.AccessAPIClient + log zerolog.Logger + nodeCommunicator Communicator + txResultCache *lru.Cache[flow.Identifier, *access.TransactionResult] + txResultQueryMode IndexQueryMode systemTxID flow.Identifier systemTx *flow.TransactionBody @@ -405,7 +411,15 @@ func (b *backendTransactions) getTransactionResultsByBlockIDFromExecutionNode( BlockId: blockID[:], } - execNodes, err := executionNodesForBlockID(ctx, blockID, b.executionReceipts, b.state, b.log) + execNodes, err := commonrpc.ExecutionNodesForBlockID( + ctx, + blockID, + b.executionReceipts, + b.state, + b.log, + preferredENIdentifiers, + fixedENIdentifiers, + ) if err != nil { if IsInsufficientExecutionReceipts(err) { return nil, status.Errorf(codes.NotFound, err.Error()) @@ -559,7 +573,15 @@ func (b *backendTransactions) getTransactionResultByIndexFromExecutionNode( Index: index, } - execNodes, err := executionNodesForBlockID(ctx, blockID, b.executionReceipts, b.state, b.log) + execNodes, err := commonrpc.ExecutionNodesForBlockID( + ctx, + blockID, + b.executionReceipts, + b.state, + b.log, + preferredENIdentifiers, + fixedENIdentifiers, + ) if err != nil { if IsInsufficientExecutionReceipts(err) { return nil, status.Errorf(codes.NotFound, err.Error()) @@ -743,7 +765,15 @@ func (b *backendTransactions) getTransactionResultFromExecutionNode( TransactionId: transactionID[:], } - execNodes, err := executionNodesForBlockID(ctx, blockID, b.executionReceipts, b.state, b.log) + execNodes, err := commonrpc.ExecutionNodesForBlockID( + ctx, + blockID, + b.executionReceipts, + b.state, + b.log, + preferredENIdentifiers, + fixedENIdentifiers, + ) if err != nil { // if no execution receipt were found, return a NotFound GRPC error if IsInsufficientExecutionReceipts(err) { @@ -962,28 +992,34 @@ func (b *backendTransactions) tryGetTransactionResultByIndex( } // LookupErrorMessageByTransactionID returns transaction error message for specified transaction. -// If an error message for transaction can be found in the cache then it will be used to serve the request, otherwise -// an RPC call will be made to the EN to fetch that error message, fetched value will be cached in the LRU cache. +// If transaction error messages are stored locally, they will be checked first in local storage. +// If error messages are not stored locally, an RPC call will be made to the EN to fetch message. +// // Expected errors during normal operation: -// - InsufficientExecutionReceipts - found insufficient receipts for given block ID. +// - InsufficientExecutionReceipts - found insufficient receipts for the given block ID. // - status.Error - remote GRPC call to EN has failed. func (b *backendTransactions) LookupErrorMessageByTransactionID( ctx context.Context, blockID flow.Identifier, + height uint64, transactionID flow.Identifier, ) (string, error) { - var cacheKey flow.Identifier - var value string - - if b.txErrorMessagesCache != nil { - cacheKey = flow.MakeIDFromFingerPrint(append(blockID[:], transactionID[:]...)) - value, cached := b.txErrorMessagesCache.Get(cacheKey) - if cached { - return value, nil + if b.txResultErrorMessages != nil { + res, err := b.txResultErrorMessages.ByBlockIDTransactionID(blockID, transactionID) + if err == nil { + return res.ErrorMessage, nil } } - execNodes, err := executionNodesForBlockID(ctx, blockID, b.executionReceipts, b.state, b.log) + execNodes, err := commonrpc.ExecutionNodesForBlockID( + ctx, + blockID, + b.executionReceipts, + b.state, + b.log, + preferredENIdentifiers, + fixedENIdentifiers, + ) if err != nil { if IsInsufficientExecutionReceipts(err) { return "", status.Errorf(codes.NotFound, err.Error()) @@ -997,23 +1033,30 @@ func (b *backendTransactions) LookupErrorMessageByTransactionID( resp, err := b.getTransactionErrorMessageFromAnyEN(ctx, execNodes, req) if err != nil { - return "", fmt.Errorf("could not fetch error message from ENs: %w", err) - } - value = resp.ErrorMessage + // If no execution nodes return a valid response, + // return a static message "failed". + txResult, err := b.txResultsIndex.ByBlockIDTransactionID(blockID, height, transactionID) + if err != nil { + return "", rpc.ConvertStorageError(err) + } - if b.txErrorMessagesCache != nil { - b.txErrorMessagesCache.Add(cacheKey, value) + if txResult.Failed { + return DefaultFailedErrorMessage, nil + } + + // in case tx result is not failed + return "", nil } - return value, nil + return resp.ErrorMessage, nil } -// LookupErrorMessageByIndex returns transaction error message for specified transaction using its index. -// If an error message for transaction can be found in cache then it will be used to serve the request, otherwise -// an RPC call will be made to the EN to fetch that error message, fetched value will be cached in the LRU cache. +// LookupErrorMessageByIndex returns the transaction error message for a specified transaction using its index. +// If transaction error messages are stored locally, they will be checked first in local storage. +// If error messages are not stored locally, an RPC call will be made to the EN to fetch message. +// // Expected errors during normal operation: -// - status.Error[codes.NotFound] - transaction result for given block ID and tx index is not available. -// - InsufficientExecutionReceipts - found insufficient receipts for given block ID. +// - InsufficientExecutionReceipts - found insufficient receipts for the given block ID. // - status.Error - remote GRPC call to EN has failed. func (b *backendTransactions) LookupErrorMessageByIndex( ctx context.Context, @@ -1021,23 +1064,22 @@ func (b *backendTransactions) LookupErrorMessageByIndex( height uint64, index uint32, ) (string, error) { - txResult, err := b.txResultsIndex.ByBlockIDTransactionIndex(blockID, height, index) - if err != nil { - return "", rpc.ConvertStorageError(err) - } - - var cacheKey flow.Identifier - var value string - - if b.txErrorMessagesCache != nil { - cacheKey = flow.MakeIDFromFingerPrint(append(blockID[:], txResult.TransactionID[:]...)) - value, cached := b.txErrorMessagesCache.Get(cacheKey) - if cached { - return value, nil + if b.txResultErrorMessages != nil { + res, err := b.txResultErrorMessages.ByBlockIDTransactionIndex(blockID, index) + if err == nil { + return res.ErrorMessage, nil } } - execNodes, err := executionNodesForBlockID(ctx, blockID, b.executionReceipts, b.state, b.log) + execNodes, err := commonrpc.ExecutionNodesForBlockID( + ctx, + blockID, + b.executionReceipts, + b.state, + b.log, + preferredENIdentifiers, + fixedENIdentifiers, + ) if err != nil { if IsInsufficientExecutionReceipts(err) { return "", status.Errorf(codes.NotFound, err.Error()) @@ -1051,55 +1093,57 @@ func (b *backendTransactions) LookupErrorMessageByIndex( resp, err := b.getTransactionErrorMessageByIndexFromAnyEN(ctx, execNodes, req) if err != nil { - return "", fmt.Errorf("could not fetch error message from ENs: %w", err) - } - value = resp.ErrorMessage + // If no execution nodes return a valid response, + // return a static message "failed" + txResult, err := b.txResultsIndex.ByBlockIDTransactionIndex(blockID, height, index) + if err != nil { + return "", rpc.ConvertStorageError(err) + } + + if txResult.Failed { + return DefaultFailedErrorMessage, nil + } - if b.txErrorMessagesCache != nil { - b.txErrorMessagesCache.Add(cacheKey, value) + // in case tx result is not failed + return "", nil } - return value, nil + return resp.ErrorMessage, nil } // LookupErrorMessagesByBlockID returns all error messages for failed transactions by blockID. -// An RPC call will be made to the EN to fetch missing errors messages, fetched value will be cached in the LRU cache. +// If transaction error messages are stored locally, they will be checked first in local storage. +// If error messages are not stored locally, an RPC call will be made to the EN to fetch messages. +// // Expected errors during normal operation: -// - status.Error[codes.NotFound] - transaction results for given block ID are not available. -// - InsufficientExecutionReceipts - found insufficient receipts for given block ID. +// - InsufficientExecutionReceipts - found insufficient receipts for the given block ID. // - status.Error - remote GRPC call to EN has failed. func (b *backendTransactions) LookupErrorMessagesByBlockID( ctx context.Context, blockID flow.Identifier, height uint64, ) (map[flow.Identifier]string, error) { - txResults, err := b.txResultsIndex.ByBlockID(blockID, height) - if err != nil { - return nil, rpc.ConvertStorageError(err) - } + result := make(map[flow.Identifier]string) - results := make(map[flow.Identifier]string) - - if b.txErrorMessagesCache != nil { - needToFetch := false - for _, txResult := range txResults { - if txResult.Failed { - cacheKey := flow.MakeIDFromFingerPrint(append(blockID[:], txResult.TransactionID[:]...)) - if value, ok := b.txErrorMessagesCache.Get(cacheKey); ok { - results[txResult.TransactionID] = value - } else { - needToFetch = true - } + if b.txResultErrorMessages != nil { + res, err := b.txResultErrorMessages.ByBlockID(blockID) + if err == nil { + for _, value := range res { + result[value.TransactionID] = value.ErrorMessage } - } - // all transactions were served from cache or there were no failed transactions - if !needToFetch { - return results, nil + return result, nil } } - execNodes, err := executionNodesForBlockID(ctx, blockID, b.executionReceipts, b.state, b.log) + execNodes, err := commonrpc.ExecutionNodesForBlockID(ctx, + blockID, + b.executionReceipts, + b.state, + b.log, + preferredENIdentifiers, + fixedENIdentifiers, + ) if err != nil { if IsInsufficientExecutionReceipts(err) { return nil, status.Errorf(codes.NotFound, err.Error()) @@ -1110,18 +1154,28 @@ func (b *backendTransactions) LookupErrorMessagesByBlockID( BlockId: convert.IdentifierToMessage(blockID), } - resp, err := b.getTransactionErrorMessagesFromAnyEN(ctx, execNodes, req) + resp, _, err := b.GetTransactionErrorMessagesFromAnyEN(ctx, execNodes, req) if err != nil { - return nil, fmt.Errorf("could not fetch error message from ENs: %w", err) + // If no execution nodes return a valid response, + // return a static message "failed" + txResults, err := b.txResultsIndex.ByBlockID(blockID, height) + if err != nil { + return nil, rpc.ConvertStorageError(err) + } + + for _, txResult := range txResults { + if txResult.Failed { + result[txResult.TransactionID] = DefaultFailedErrorMessage + } + } + + return result, nil } - result := make(map[flow.Identifier]string, len(resp)) + for _, value := range resp { - if b.txErrorMessagesCache != nil { - cacheKey := flow.MakeIDFromFingerPrint(append(req.BlockId, value.TransactionId...)) - b.txErrorMessagesCache.Add(cacheKey, value.ErrorMessage) - } result[convert.MessageToIdentifier(value.TransactionId)] = value.ErrorMessage } + return result, nil } @@ -1209,26 +1263,29 @@ func (b *backendTransactions) getTransactionErrorMessageByIndexFromAnyEN( return resp, nil } -// getTransactionErrorMessagesFromAnyEN performs an RPC call using available nodes passed as argument. List of nodes must be non-empty otherwise an error will be returned. +// GetTransactionErrorMessagesFromAnyEN performs an RPC call using available nodes passed as argument. List of nodes must be non-empty otherwise an error will be returned. // Expected errors during normal operation: // - status.Error - GRPC call failed, some of possible codes are: // - codes.NotFound - request cannot be served by EN because of absence of data. // - codes.Unavailable - remote node is not unavailable. -func (b *backendTransactions) getTransactionErrorMessagesFromAnyEN( +func (b *backendTransactions) GetTransactionErrorMessagesFromAnyEN( ctx context.Context, execNodes flow.IdentitySkeletonList, req *execproto.GetTransactionErrorMessagesByBlockIDRequest, -) ([]*execproto.GetTransactionErrorMessagesResponse_Result, error) { +) ([]*execproto.GetTransactionErrorMessagesResponse_Result, *flow.IdentitySkeleton, error) { // if we were passed 0 execution nodes add a specific error if len(execNodes) == 0 { - return nil, errors.New("zero execution nodes") + return nil, nil, errors.New("zero execution nodes") } var resp *execproto.GetTransactionErrorMessagesResponse + var execNode *flow.IdentitySkeleton + errToReturn := b.nodeCommunicator.CallAvailableNode( execNodes, func(node *flow.IdentitySkeleton) error { var err error + execNode = node resp, err = b.tryGetTransactionErrorMessagesByBlockIDFromEN(ctx, node, req) if err == nil { b.log.Debug(). @@ -1245,10 +1302,10 @@ func (b *backendTransactions) getTransactionErrorMessagesFromAnyEN( // log the errors if errToReturn != nil { b.log.Err(errToReturn).Msg("failed to get transaction error messages from execution nodes") - return nil, errToReturn + return nil, nil, errToReturn } - return resp.GetResults(), nil + return resp.GetResults(), execNode, nil } // Expected errors during normal operation: diff --git a/engine/access/rpc/backend/backend_transactions_test.go b/engine/access/rpc/backend/backend_transactions_test.go index 5d1c513cef8..dabdd33bbbf 100644 --- a/engine/access/rpc/backend/backend_transactions_test.go +++ b/engine/access/rpc/backend/backend_transactions_test.go @@ -338,96 +338,191 @@ func (suite *Suite) TestGetTransactionResultUnknownFromCache() { }) } -// TestLookupTransactionErrorMessage_HappyPath tests lookup of a transaction error message. In a happy path if it wasn't found in the cache, it -// has to be fetched from the execution node, otherwise served from the cache. -// If the transaction has not failed, the error message must be empty. -func (suite *Suite) TestLookupTransactionErrorMessage_HappyPath() { +// TestLookupTransactionErrorMessageByTransactionID_HappyPath verifies the lookup of a transaction error message +// by block id and transaction id. +// It tests two cases: +// 1. Happy path where the error message is fetched from the EN if it's not found in the cache. +// 2. Happy path where the error message is served from the storage database if it exists. +func (suite *Suite) TestLookupTransactionErrorMessageByTransactionID_HappyPath() { block := unittest.BlockFixture() blockId := block.ID() failedTx := unittest.TransactionFixture() failedTxId := failedTx.ID() + failedTxIndex := rand.Uint32() + // Setup mock receipts and execution node identities. _, fixedENIDs := suite.setupReceipts(&block) suite.state.On("Final").Return(suite.snapshot, nil).Maybe() suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) - // create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) - params := suite.defaultBackendParams() - // the connection factory should be used to get the execution node client - params.ConnFactory = connFactory - params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() + params.TxResultErrorMessages = suite.txErrorMessages - backend, err := New(params) - suite.Require().NoError(err) + // Test case: transaction error message is fetched from the EN. + suite.Run("happy path from EN", func() { + // the connection factory should be used to get the execution node client + params.ConnFactory = suite.setupConnectionFactory() + params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() - expectedErrorMsg := "some error" + // Mock the cache lookup for the transaction error message, returning "not found". + suite.txErrorMessages.On("ByBlockIDTransactionID", blockId, failedTxId). + Return(nil, storage.ErrNotFound).Once() - exeEventReq := &execproto.GetTransactionErrorMessageRequest{ - BlockId: blockId[:], - TransactionId: failedTxId[:], - } + backend, err := New(params) + suite.Require().NoError(err) - exeEventResp := &execproto.GetTransactionErrorMessageResponse{ - TransactionId: failedTxId[:], - ErrorMessage: expectedErrorMsg, - } + // Mock the execution node API call to fetch the error message. + exeEventReq := &execproto.GetTransactionErrorMessageRequest{ + BlockId: blockId[:], + TransactionId: failedTxId[:], + } + exeEventResp := &execproto.GetTransactionErrorMessageResponse{ + TransactionId: failedTxId[:], + ErrorMessage: expectedErrorMsg, + } + suite.execClient.On("GetTransactionErrorMessage", mock.Anything, exeEventReq).Return(exeEventResp, nil).Once() - suite.execClient.On("GetTransactionErrorMessage", mock.Anything, exeEventReq).Return(exeEventResp, nil).Once() + // Perform the lookup and assert that the error message is retrieved correctly. + errMsg, err := backend.LookupErrorMessageByTransactionID(context.Background(), blockId, block.Header.Height, failedTxId) + suite.Require().NoError(err) + suite.Require().Equal(expectedErrorMsg, errMsg) + suite.assertAllExpectations() + }) - errMsg, err := backend.LookupErrorMessageByTransactionID(context.Background(), blockId, failedTxId) - suite.Require().NoError(err) - suite.Require().Equal(expectedErrorMsg, errMsg) + // Test case: transaction error message is fetched from the storage database. + suite.Run("happy path from storage db", func() { + backend, err := New(params) + suite.Require().NoError(err) - // ensure the transaction error message is cached after retrieval; we do this by mocking the grpc call - // only once - errMsg, err = backend.LookupErrorMessageByTransactionID(context.Background(), blockId, failedTxId) - suite.Require().NoError(err) - suite.Require().Equal(expectedErrorMsg, errMsg) - suite.assertAllExpectations() + // Mock the cache lookup for the transaction error message, returning a stored result. + suite.txErrorMessages.On("ByBlockIDTransactionID", blockId, failedTxId). + Return(&flow.TransactionResultErrorMessage{ + TransactionID: failedTxId, + ErrorMessage: expectedErrorMsg, + Index: failedTxIndex, + ExecutorID: unittest.IdentifierFixture(), + }, nil).Once() + + // Perform the lookup and assert that the error message is retrieved correctly from storage. + errMsg, err := backend.LookupErrorMessageByTransactionID(context.Background(), blockId, block.Header.Height, failedTxId) + suite.Require().NoError(err) + suite.Require().Equal(expectedErrorMsg, errMsg) + suite.assertAllExpectations() + }) } -// TestLookupTransactionErrorMessage_FailedToFetch tests lookup of a transaction error message, when a transaction result -// is not in the cache and needs to be fetched from EN, but the EN fails to return it. -func (suite *Suite) TestLookupTransactionErrorMessage_FailedToFetch() { +// TestLookupTransactionErrorMessageByTransactionID_FailedToFetch tests the case when a transaction error message +// is not in the cache and needs to be fetched from the EN, but the EN fails to return it. +// It tests three cases: +// 1. The transaction is not found in the transaction results, leading to a "NotFound" error. +// 2. The transaction result is not failed, and the error message is empty. +// 3. The transaction result is failed, and the error message "failed" are returned. +func (suite *Suite) TestLookupTransactionErrorMessageByTransactionID_FailedToFetch() { block := unittest.BlockFixture() blockId := block.ID() failedTx := unittest.TransactionFixture() failedTxId := failedTx.ID() + // Setup mock receipts and execution node identities. _, fixedENIDs := suite.setupReceipts(&block) suite.state.On("Final").Return(suite.snapshot, nil).Maybe() suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) - // create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) + // Create a mock index reporter + reporter := syncmock.NewIndexReporter(suite.T()) + reporter.On("LowestIndexedHeight").Return(block.Header.Height, nil) + reporter.On("HighestIndexedHeight").Return(block.Header.Height+10, nil) params := suite.defaultBackendParams() - // the connection factory should be used to get the execution node client - params.ConnFactory = connFactory + // The connection factory should be used to get the execution node client + params.ConnFactory = suite.setupConnectionFactory() + // Initialize the transaction results index with the mock reporter. params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() + params.TxResultsIndex = index.NewTransactionResultsIndex(index.NewReporter(), suite.transactionResults) + err := params.TxResultsIndex.Initialize(reporter) + suite.Require().NoError(err) + + params.TxResultErrorMessages = suite.txErrorMessages backend, err := New(params) suite.Require().NoError(err) - // lookup should try each of the 2 ENs in fixedENIDs - suite.execClient.On("GetTransactionErrorMessage", mock.Anything, mock.Anything).Return(nil, - status.Error(codes.Unavailable, "")).Twice() + // Test case: failed to fetch from EN, transaction is unknown. + suite.Run("failed to fetch from EN, unknown tx", func() { + // lookup should try each of the 2 ENs in fixedENIDs + suite.execClient.On("GetTransactionErrorMessage", mock.Anything, mock.Anything).Return(nil, + status.Error(codes.Unavailable, "")).Twice() + + // Setup mock that the transaction and tx error message is not found in the storage. + suite.txErrorMessages.On("ByBlockIDTransactionID", blockId, failedTxId). + Return(nil, storage.ErrNotFound).Once() + suite.transactionResults.On("ByBlockIDTransactionID", blockId, failedTxId). + Return(nil, storage.ErrNotFound).Once() - errMsg, err := backend.LookupErrorMessageByTransactionID(context.Background(), blockId, failedTxId) - suite.Require().Error(err) - suite.Require().Equal(codes.Unavailable, status.Code(err)) - suite.Require().Empty(errMsg) + // Perform the lookup and expect a "NotFound" error with an empty error message. + errMsg, err := backend.LookupErrorMessageByTransactionID(context.Background(), blockId, block.Header.Height, failedTxId) + suite.Require().Error(err) + suite.Require().Equal(codes.NotFound, status.Code(err)) + suite.Require().Empty(errMsg) + suite.assertAllExpectations() + }) - suite.assertAllExpectations() + // Test case: failed to fetch from EN, but the transaction result is not failed. + suite.Run("failed to fetch from EN, tx result is not failed", func() { + // Lookup should try each of the 2 ENs in fixedENIDs + suite.execClient.On("GetTransactionErrorMessage", mock.Anything, mock.Anything).Return(nil, + status.Error(codes.Unavailable, "")).Twice() + + // Setup mock that the transaction error message is not found in storage. + suite.txErrorMessages.On("ByBlockIDTransactionID", blockId, failedTxId). + Return(nil, storage.ErrNotFound).Once() + + // Setup mock that the transaction result exists and is not failed. + suite.transactionResults.On("ByBlockIDTransactionID", blockId, failedTxId). + Return(&flow.LightTransactionResult{ + TransactionID: failedTxId, + Failed: false, + ComputationUsed: 0, + }, nil).Once() + + // Perform the lookup and expect no error and an empty error message. + errMsg, err := backend.LookupErrorMessageByTransactionID(context.Background(), blockId, block.Header.Height, failedTxId) + suite.Require().NoError(err) + suite.Require().Empty(errMsg) + suite.assertAllExpectations() + }) + + // Test case: failed to fetch from EN, but the transaction result is failed. + suite.Run("failed to fetch from EN, tx result is failed", func() { + // lookup should try each of the 2 ENs in fixedENIDs + suite.execClient.On("GetTransactionErrorMessage", mock.Anything, mock.Anything).Return(nil, + status.Error(codes.Unavailable, "")).Twice() + + // Setup mock that the transaction error message is not found in storage. + suite.txErrorMessages.On("ByBlockIDTransactionID", blockId, failedTxId). + Return(nil, storage.ErrNotFound).Once() + + // Setup mock that the transaction result exists and is failed. + suite.transactionResults.On("ByBlockIDTransactionID", blockId, failedTxId). + Return(&flow.LightTransactionResult{ + TransactionID: failedTxId, + Failed: true, + ComputationUsed: 0, + }, nil).Once() + + // Perform the lookup and expect the failed error message to be returned. + errMsg, err := backend.LookupErrorMessageByTransactionID(context.Background(), blockId, block.Header.Height, failedTxId) + suite.Require().NoError(err) + suite.Require().Equal(errMsg, DefaultFailedErrorMessage) + suite.assertAllExpectations() + }) } -// TestLookupTransactionErrorMessageByIndex_HappyPath tests lookup of a transaction error message by index. -// In a happy path if it wasn't found in the cache, it has to be fetched from the execution node, otherwise served from the cache. -// If the transaction has not failed, the error message must be empty. +// TestLookupTransactionErrorMessageByIndex_HappyPath verifies the lookup of a transaction error message +// by block ID and transaction index. +// It tests two cases: +// 1. Happy path where the error message is fetched from the EN if it is not found in the cache. +// 2. Happy path where the error message is served from the storage database if it exists. func (suite *Suite) TestLookupTransactionErrorMessageByIndex_HappyPath() { block := unittest.BlockFixture() blockId := block.ID() @@ -435,122 +530,90 @@ func (suite *Suite) TestLookupTransactionErrorMessageByIndex_HappyPath() { failedTxId := failedTx.ID() failedTxIndex := rand.Uint32() - suite.transactionResults.On("ByBlockIDTransactionIndex", blockId, failedTxIndex). - Return(&flow.LightTransactionResult{ - TransactionID: failedTxId, - Failed: true, - ComputationUsed: 0, - }, nil).Twice() - + // Setup mock receipts and execution node identities. _, fixedENIDs := suite.setupReceipts(&block) suite.state.On("Final").Return(suite.snapshot, nil).Maybe() suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) - // create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) - - // create a mock index reporter - reporter := syncmock.NewIndexReporter(suite.T()) - reporter.On("LowestIndexedHeight").Return(block.Header.Height, nil) - reporter.On("HighestIndexedHeight").Return(block.Header.Height+10, nil) - params := suite.defaultBackendParams() + params.TxResultErrorMessages = suite.txErrorMessages - // the connection factory should be used to get the execution node client - params.ConnFactory = connFactory - params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() - - params.TxResultsIndex = index.NewTransactionResultsIndex(index.NewReporter(), suite.transactionResults) - err := params.TxResultsIndex.Initialize(reporter) - suite.Require().NoError(err) - - backend, err := New(params) - suite.Require().NoError(err) - - expectedErrorMsg := "some error" - - exeEventReq := &execproto.GetTransactionErrorMessageByIndexRequest{ - BlockId: blockId[:], - Index: failedTxIndex, - } - - exeEventResp := &execproto.GetTransactionErrorMessageResponse{ - TransactionId: failedTxId[:], - ErrorMessage: expectedErrorMsg, - } - - suite.execClient.On("GetTransactionErrorMessageByIndex", mock.Anything, exeEventReq).Return(exeEventResp, nil).Once() - - errMsg, err := backend.LookupErrorMessageByIndex(context.Background(), blockId, block.Header.Height, failedTxIndex) - suite.Require().NoError(err) - suite.Require().Equal(expectedErrorMsg, errMsg) - - // ensure the transaction error message is cached after retrieval; we do this by mocking the grpc call - // only once - errMsg, err = backend.LookupErrorMessageByIndex(context.Background(), blockId, block.Header.Height, failedTxIndex) - suite.Require().NoError(err) - suite.Require().Equal(expectedErrorMsg, errMsg) - suite.assertAllExpectations() -} - -// TestLookupTransactionErrorMessageByIndex_UnknownTransaction tests lookup of a transaction error message by index, -// when a transaction result has not been synced yet, in this case nothing we can do but return an error. -func (suite *Suite) TestLookupTransactionErrorMessageByIndex_UnknownTransaction() { - block := unittest.BlockFixture() - blockId := block.ID() - failedTxIndex := rand.Uint32() + // Test case: transaction error message is fetched from the EN. + suite.Run("happy path from EN", func() { + // the connection factory should be used to get the execution node client + params.ConnFactory = suite.setupConnectionFactory() + params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() - suite.transactionResults.On("ByBlockIDTransactionIndex", blockId, failedTxIndex). - Return(nil, storage.ErrNotFound).Once() + // Mock the cache lookup for the transaction error message, returning "not found". + suite.txErrorMessages.On("ByBlockIDTransactionIndex", blockId, failedTxIndex). + Return(nil, storage.ErrNotFound).Once() - // create a mock index reporter - reporter := syncmock.NewIndexReporter(suite.T()) - reporter.On("LowestIndexedHeight").Return(block.Header.Height, nil) - reporter.On("HighestIndexedHeight").Return(block.Header.Height+10, nil) - - params := suite.defaultBackendParams() + backend, err := New(params) + suite.Require().NoError(err) - params.TxResultsIndex = index.NewTransactionResultsIndex(index.NewReporter(), suite.transactionResults) - err := params.TxResultsIndex.Initialize(reporter) - suite.Require().NoError(err) + // Mock the execution node API call to fetch the error message. + exeEventReq := &execproto.GetTransactionErrorMessageByIndexRequest{ + BlockId: blockId[:], + Index: failedTxIndex, + } + exeEventResp := &execproto.GetTransactionErrorMessageResponse{ + TransactionId: failedTxId[:], + ErrorMessage: expectedErrorMsg, + } + suite.execClient.On("GetTransactionErrorMessageByIndex", mock.Anything, exeEventReq).Return(exeEventResp, nil).Once() - backend, err := New(params) - suite.Require().NoError(err) + // Perform the lookup and assert that the error message is retrieved correctly. + errMsg, err := backend.LookupErrorMessageByIndex(context.Background(), blockId, block.Header.Height, failedTxIndex) + suite.Require().NoError(err) + suite.Require().Equal(expectedErrorMsg, errMsg) + suite.assertAllExpectations() + }) - errMsg, err := backend.LookupErrorMessageByIndex(context.Background(), blockId, block.Header.Height, failedTxIndex) - suite.Require().Error(err) - suite.Require().Equal(codes.NotFound, status.Code(err)) - suite.Require().Empty(errMsg) + // Test case: transaction error message is fetched from the storage database. + suite.Run("happy path from storage db", func() { + backend, err := New(params) + suite.Require().NoError(err) - suite.assertAllExpectations() + // Mock the cache lookup for the transaction error message, returning a stored result. + suite.txErrorMessages.On("ByBlockIDTransactionIndex", blockId, failedTxIndex). + Return(&flow.TransactionResultErrorMessage{ + TransactionID: failedTxId, + ErrorMessage: expectedErrorMsg, + Index: failedTxIndex, + ExecutorID: unittest.IdentifierFixture(), + }, nil).Once() + + // Perform the lookup and assert that the error message is retrieved correctly from storage. + errMsg, err := backend.LookupErrorMessageByIndex(context.Background(), blockId, block.Header.Height, failedTxIndex) + suite.Require().NoError(err) + suite.Require().Equal(expectedErrorMsg, errMsg) + suite.assertAllExpectations() + }) } -// TestLookupTransactionErrorMessageByIndex_FailedToFetch tests lookup of a transaction error message by index, -// when a transaction result is not in the cache and needs to be fetched from EN, but the EN fails to return it. +// TestLookupTransactionErrorMessageByIndex_FailedToFetch verifies the behavior of looking up a transaction error message by index +// when the error message is not in the cache, and fetching it from the EN fails. +// It tests three cases: +// 1. The transaction is not found in the transaction results, leading to a "NotFound" error. +// 2. The transaction result is not failed, and the error message is empty. +// 3. The transaction result is failed, and the error message "failed" are returned. func (suite *Suite) TestLookupTransactionErrorMessageByIndex_FailedToFetch() { block := unittest.BlockFixture() blockId := block.ID() + failedTxIndex := rand.Uint32() failedTx := unittest.TransactionFixture() failedTxId := failedTx.ID() - failedTxIndex := rand.Uint32() - - suite.transactionResults.On("ByBlockIDTransactionIndex", blockId, failedTxIndex). - Return(&flow.LightTransactionResult{ - TransactionID: failedTxId, - Failed: true, - ComputationUsed: 0, - }, nil).Once() + // Setup mock receipts and execution node identities. _, fixedENIDs := suite.setupReceipts(&block) suite.state.On("Final").Return(suite.snapshot, nil).Maybe() suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) - // create a mock connection factory + // Create a mock connection factory connFactory := connectionmock.NewConnectionFactory(suite.T()) connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) - // create a mock index reporter + // Create a mock index reporter reporter := syncmock.NewIndexReporter(suite.T()) reporter.On("LowestIndexedHeight").Return(block.Header.Height, nil) reporter.On("HighestIndexedHeight").Return(block.Header.Height+10, nil) @@ -559,30 +622,92 @@ func (suite *Suite) TestLookupTransactionErrorMessageByIndex_FailedToFetch() { // the connection factory should be used to get the execution node client params.ConnFactory = connFactory params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() - + // Initialize the transaction results index with the mock reporter. params.TxResultsIndex = index.NewTransactionResultsIndex(index.NewReporter(), suite.transactionResults) err := params.TxResultsIndex.Initialize(reporter) suite.Require().NoError(err) + params.TxResultErrorMessages = suite.txErrorMessages + backend, err := New(params) suite.Require().NoError(err) - // lookup should try each of the 2 ENs in fixedENIDs - suite.execClient.On("GetTransactionErrorMessageByIndex", mock.Anything, mock.Anything).Return(nil, - status.Error(codes.Unavailable, "")).Twice() + // Test case: failed to fetch from EN, transaction is unknown. + suite.Run("failed to fetch from EN, unknown tx", func() { + // lookup should try each of the 2 ENs in fixedENIDs + suite.execClient.On("GetTransactionErrorMessageByIndex", mock.Anything, mock.Anything).Return(nil, + status.Error(codes.Unavailable, "")).Twice() - errMsg, err := backend.LookupErrorMessageByIndex(context.Background(), blockId, block.Header.Height, failedTxIndex) - suite.Require().Error(err) - suite.Require().Equal(codes.Unavailable, status.Code(err)) - suite.Require().Empty(errMsg) + // Setup mock that the transaction and tx error message is not found in the storage. + suite.txErrorMessages.On("ByBlockIDTransactionIndex", blockId, failedTxIndex). + Return(nil, storage.ErrNotFound).Once() + suite.transactionResults.On("ByBlockIDTransactionIndex", blockId, failedTxIndex). + Return(nil, storage.ErrNotFound).Once() - suite.assertAllExpectations() + // Perform the lookup and expect a "NotFound" error with an empty error message. + errMsg, err := backend.LookupErrorMessageByIndex(context.Background(), blockId, block.Header.Height, failedTxIndex) + suite.Require().Error(err) + suite.Require().Equal(codes.NotFound, status.Code(err)) + suite.Require().Empty(errMsg) + suite.assertAllExpectations() + }) + + // Test case: failed to fetch from EN, but the transaction result is not failed. + suite.Run("failed to fetch from EN, tx result is not failed", func() { + // lookup should try each of the 2 ENs in fixedENIDs + suite.execClient.On("GetTransactionErrorMessageByIndex", mock.Anything, mock.Anything).Return(nil, + status.Error(codes.Unavailable, "")).Twice() + + // Setup mock that the transaction error message is not found in storage. + suite.txErrorMessages.On("ByBlockIDTransactionIndex", blockId, failedTxIndex). + Return(nil, storage.ErrNotFound).Once() + + // Setup mock that the transaction result exists and is not failed. + suite.transactionResults.On("ByBlockIDTransactionIndex", blockId, failedTxIndex). + Return(&flow.LightTransactionResult{ + TransactionID: failedTxId, + Failed: false, + ComputationUsed: 0, + }, nil).Once() + + // Perform the lookup and expect no error and an empty error message. + errMsg, err := backend.LookupErrorMessageByIndex(context.Background(), blockId, block.Header.Height, failedTxIndex) + suite.Require().NoError(err) + suite.Require().Empty(errMsg) + suite.assertAllExpectations() + }) + + // Test case: failed to fetch from EN, but the transaction result is failed. + suite.Run("failed to fetch from EN, tx result is failed", func() { + // lookup should try each of the 2 ENs in fixedENIDs + suite.execClient.On("GetTransactionErrorMessageByIndex", mock.Anything, mock.Anything).Return(nil, + status.Error(codes.Unavailable, "")).Twice() + + // Setup mock that the transaction error message is not found in storage. + suite.txErrorMessages.On("ByBlockIDTransactionIndex", blockId, failedTxIndex). + Return(nil, storage.ErrNotFound).Once() + + // Setup mock that the transaction result exists and is failed. + suite.transactionResults.On("ByBlockIDTransactionIndex", blockId, failedTxIndex). + Return(&flow.LightTransactionResult{ + TransactionID: failedTxId, + Failed: true, + ComputationUsed: 0, + }, nil).Once() + + // Perform the lookup and expect the failed error message to be returned. + errMsg, err := backend.LookupErrorMessageByIndex(context.Background(), blockId, block.Header.Height, failedTxIndex) + suite.Require().NoError(err) + suite.Require().Equal(errMsg, DefaultFailedErrorMessage) + suite.assertAllExpectations() + }) } -// TestLookupTransactionErrorMessages_HappyPath tests lookup of a transaction error messages by block ID. -// In a happy path, it has to be fetched from the execution node if there are no cached results. -// All fetched transactions have to be cached for future calls. -func (suite *Suite) TestLookupTransactionErrorMessages_HappyPath() { +// TestLookupTransactionErrorMessagesByBlockID_HappyPath verifies the lookup of transaction error messages by block ID. +// It tests two cases: +// 1. Happy path where the error messages are fetched from the EN if they are not found in the cache. +// 2. Happy path where the error messages are served from the storage database if they exist. +func (suite *Suite) TestLookupTransactionErrorMessagesByBlockID_HappyPath() { block := unittest.BlockFixture() blockId := block.ID() @@ -595,211 +720,225 @@ func (suite *Suite) TestLookupTransactionErrorMessages_HappyPath() { }) } - suite.transactionResults.On("ByBlockID", blockId). - Return(resultsByBlockID, nil).Twice() - _, fixedENIDs := suite.setupReceipts(&block) suite.state.On("Final").Return(suite.snapshot, nil).Maybe() suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) - // create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) - - // create a mock index reporter - reporter := syncmock.NewIndexReporter(suite.T()) - reporter.On("LowestIndexedHeight").Return(block.Header.Height, nil) - reporter.On("HighestIndexedHeight").Return(block.Header.Height+10, nil) - params := suite.defaultBackendParams() + params.TxResultErrorMessages = suite.txErrorMessages - // the connection factory should be used to get the execution node client - params.ConnFactory = connFactory - params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() - - params.TxResultsIndex = index.NewTransactionResultsIndex(index.NewReporter(), suite.transactionResults) - err := params.TxResultsIndex.Initialize(reporter) - suite.Require().NoError(err) + // Test case: transaction error messages is fetched from the EN. + suite.Run("happy path from EN", func() { + // the connection factory should be used to get the execution node client + params.ConnFactory = suite.setupConnectionFactory() + params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() - backend, err := New(params) - suite.Require().NoError(err) + // Mock the cache lookup for the transaction error messages, returning "not found". + suite.txErrorMessages.On("ByBlockID", blockId). + Return(nil, storage.ErrNotFound).Once() - expectedErrorMsg := "some error" + backend, err := New(params) + suite.Require().NoError(err) - exeEventReq := &execproto.GetTransactionErrorMessagesByBlockIDRequest{ - BlockId: blockId[:], - } + // Mock the execution node API call to fetch the error messages. + exeEventReq := &execproto.GetTransactionErrorMessagesByBlockIDRequest{ + BlockId: blockId[:], + } + exeErrMessagesResp := &execproto.GetTransactionErrorMessagesResponse{} + for _, result := range resultsByBlockID { + r := result + if r.Failed { + errMsg := fmt.Sprintf("%s.%s", expectedErrorMsg, r.TransactionID) + exeErrMessagesResp.Results = append(exeErrMessagesResp.Results, &execproto.GetTransactionErrorMessagesResponse_Result{ + TransactionId: r.TransactionID[:], + ErrorMessage: errMsg, + }) + } + } + suite.execClient.On("GetTransactionErrorMessagesByBlockID", mock.Anything, exeEventReq). + Return(exeErrMessagesResp, nil). + Once() - exeEventResp := &execproto.GetTransactionErrorMessagesResponse{} - for _, result := range resultsByBlockID { - r := result - if r.Failed { - errMsg := fmt.Sprintf("%s.%s", expectedErrorMsg, r.TransactionID) - exeEventResp.Results = append(exeEventResp.Results, &execproto.GetTransactionErrorMessagesResponse_Result{ - TransactionId: r.TransactionID[:], - ErrorMessage: errMsg, - }) + // Perform the lookup and assert that the error message is retrieved correctly. + errMessages, err := backend.LookupErrorMessagesByBlockID(context.Background(), blockId, block.Header.Height) + suite.Require().NoError(err) + suite.Require().Len(errMessages, len(exeErrMessagesResp.Results)) + for _, expectedResult := range exeErrMessagesResp.Results { + errMsg, ok := errMessages[convert.MessageToIdentifier(expectedResult.TransactionId)] + suite.Require().True(ok) + suite.Assert().Equal(expectedResult.ErrorMessage, errMsg) } - } + suite.assertAllExpectations() + }) - suite.execClient.On("GetTransactionErrorMessagesByBlockID", mock.Anything, exeEventReq). - Return(exeEventResp, nil). - Once() + // Test case: transaction error messages is fetched from the storage database. + suite.Run("happy path from storage db", func() { + backend, err := New(params) + suite.Require().NoError(err) - errMessages, err := backend.LookupErrorMessagesByBlockID(context.Background(), blockId, block.Header.Height) - suite.Require().NoError(err) - suite.Require().Len(errMessages, len(exeEventResp.Results)) - for _, expectedResult := range exeEventResp.Results { - errMsg, ok := errMessages[convert.MessageToIdentifier(expectedResult.TransactionId)] - suite.Require().True(ok) - suite.Assert().Equal(expectedResult.ErrorMessage, errMsg) - } + // Mock the cache lookup for the transaction error messages, returning a stored result. + var txErrorMessages []flow.TransactionResultErrorMessage + for i, result := range resultsByBlockID { + if result.Failed { + errMsg := fmt.Sprintf("%s.%s", expectedErrorMsg, result.TransactionID) + + txErrorMessages = append(txErrorMessages, + flow.TransactionResultErrorMessage{ + TransactionID: result.TransactionID, + ErrorMessage: errMsg, + Index: uint32(i), + ExecutorID: unittest.IdentifierFixture(), + }) + } + } + suite.txErrorMessages.On("ByBlockID", blockId). + Return(txErrorMessages, nil).Once() - // ensure the transaction error message is cached after retrieval; we do this by mocking the grpc call - // only once - errMessages, err = backend.LookupErrorMessagesByBlockID(context.Background(), blockId, block.Header.Height) - suite.Require().NoError(err) - suite.Require().Len(errMessages, len(exeEventResp.Results)) - for _, expectedResult := range exeEventResp.Results { - errMsg, ok := errMessages[convert.MessageToIdentifier(expectedResult.TransactionId)] - suite.Require().True(ok) - suite.Assert().Equal(expectedResult.ErrorMessage, errMsg) - } - suite.assertAllExpectations() + // Perform the lookup and assert that the error message is retrieved correctly from storage. + errMessages, err := backend.LookupErrorMessagesByBlockID(context.Background(), blockId, block.Header.Height) + suite.Require().NoError(err) + suite.Require().Len(errMessages, len(txErrorMessages)) + for _, expected := range txErrorMessages { + errMsg, ok := errMessages[expected.TransactionID] + suite.Require().True(ok) + suite.Assert().Equal(expected.ErrorMessage, errMsg) + } + suite.assertAllExpectations() + }) } -// TestLookupTransactionErrorMessages_HappyPath_NoFailedTxns tests lookup of a transaction error messages by block ID. -// In a happy path where a block with no failed txns is requested. We don't want to perform an RPC call in this case. -func (suite *Suite) TestLookupTransactionErrorMessages_HappyPath_NoFailedTxns() { +// TestLookupTransactionErrorMessagesByBlockID_FailedToFetch tests lookup of a transaction error messages by block ID, +// when a transaction result is not in the cache and needs to be fetched from EN, but the EN fails to return it. +// It tests three cases: +// 1. The transaction is not found in the transaction results, leading to a "NotFound" error. +// 2. The transaction result is not failed, and the error message is empty. +// 3. The transaction result is failed, and the error message "failed" are returned. +func (suite *Suite) TestLookupTransactionErrorMessagesByBlockID_FailedToFetch() { block := unittest.BlockFixture() blockId := block.ID() - resultsByBlockID := []flow.LightTransactionResult{ - { - TransactionID: unittest.IdentifierFixture(), - Failed: false, - ComputationUsed: 0, - }, - { - TransactionID: unittest.IdentifierFixture(), - Failed: false, - ComputationUsed: 0, - }, - } - - suite.transactionResults.On("ByBlockID", blockId). - Return(resultsByBlockID, nil).Once() + // Setup mock receipts and execution node identities. + _, fixedENIDs := suite.setupReceipts(&block) + suite.state.On("Final").Return(suite.snapshot, nil).Maybe() + suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) - // create a mock index reporter + // Create a mock index reporter reporter := syncmock.NewIndexReporter(suite.T()) reporter.On("LowestIndexedHeight").Return(block.Header.Height, nil) reporter.On("HighestIndexedHeight").Return(block.Header.Height+10, nil) params := suite.defaultBackendParams() - + // the connection factory should be used to get the execution node client + params.ConnFactory = suite.setupConnectionFactory() + params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() + // Initialize the transaction results index with the mock reporter. params.TxResultsIndex = index.NewTransactionResultsIndex(index.NewReporter(), suite.transactionResults) err := params.TxResultsIndex.Initialize(reporter) suite.Require().NoError(err) - backend, err := New(params) - suite.Require().NoError(err) - - errMessages, err := backend.LookupErrorMessagesByBlockID(context.Background(), blockId, block.Header.Height) - suite.Require().NoError(err) - suite.Require().Empty(errMessages) - suite.assertAllExpectations() -} - -// TestLookupTransactionErrorMessages_UnknownTransaction tests lookup of a transaction error messages by block ID, -// when a transaction results for block has not been synced yet, in this case nothing we can do but return an error. -func (suite *Suite) TestLookupTransactionErrorMessages_UnknownTransaction() { - block := unittest.BlockFixture() - blockId := block.ID() - - suite.transactionResults.On("ByBlockID", blockId). - Return(nil, storage.ErrNotFound).Once() - - // create a mock index reporter - reporter := syncmock.NewIndexReporter(suite.T()) - reporter.On("LowestIndexedHeight").Return(block.Header.Height, nil) - reporter.On("HighestIndexedHeight").Return(block.Header.Height+10, nil) - - params := suite.defaultBackendParams() - - params.TxResultsIndex = index.NewTransactionResultsIndex(index.NewReporter(), suite.transactionResults) - err := params.TxResultsIndex.Initialize(reporter) - suite.Require().NoError(err) + params.TxResultErrorMessages = suite.txErrorMessages backend, err := New(params) suite.Require().NoError(err) - errMsg, err := backend.LookupErrorMessagesByBlockID(context.Background(), blockId, block.Header.Height) - suite.Require().Error(err) - suite.Require().Equal(codes.NotFound, status.Code(err)) - suite.Require().Empty(errMsg) - - suite.assertAllExpectations() -} - -// TestLookupTransactionErrorMessages_FailedToFetch tests lookup of a transaction error messages by block ID, -// when a transaction result is not in the cache and needs to be fetched from EN, but the EN fails to return it. -func (suite *Suite) TestLookupTransactionErrorMessages_FailedToFetch() { - block := unittest.BlockFixture() - blockId := block.ID() - - resultsByBlockID := []flow.LightTransactionResult{ - { - TransactionID: unittest.IdentifierFixture(), - Failed: true, - ComputationUsed: 0, - }, - { - TransactionID: unittest.IdentifierFixture(), - Failed: true, - ComputationUsed: 0, - }, - } - - suite.transactionResults.On("ByBlockID", blockId). - Return(resultsByBlockID, nil).Once() + // Test case: failed to fetch from EN, transaction is unknown. + suite.Run("failed to fetch from EN, unknown tx", func() { + // lookup should try each of the 2 ENs in fixedENIDs + suite.execClient.On("GetTransactionErrorMessagesByBlockID", mock.Anything, mock.Anything).Return(nil, + status.Error(codes.Unavailable, "")).Twice() - _, fixedENIDs := suite.setupReceipts(&block) - suite.state.On("Final").Return(suite.snapshot, nil).Maybe() - suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) - - // create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) + // Setup mock that the transaction and tx error messages is not found in the storage. + suite.txErrorMessages.On("ByBlockID", blockId). + Return(nil, storage.ErrNotFound).Once() + suite.transactionResults.On("ByBlockID", blockId). + Return(nil, storage.ErrNotFound).Once() - // create a mock index reporter - reporter := syncmock.NewIndexReporter(suite.T()) - reporter.On("LowestIndexedHeight").Return(block.Header.Height, nil) - reporter.On("HighestIndexedHeight").Return(block.Header.Height+10, nil) + // Perform the lookup and expect a "NotFound" error with an empty error message. + errMsg, err := backend.LookupErrorMessagesByBlockID(context.Background(), blockId, block.Header.Height) + suite.Require().Error(err) + suite.Require().Equal(codes.NotFound, status.Code(err)) + suite.Require().Empty(errMsg) + suite.assertAllExpectations() + }) - params := suite.defaultBackendParams() - // the connection factory should be used to get the execution node client - params.ConnFactory = connFactory - params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() + // Test case: failed to fetch from EN, but the transaction result is not failed. + suite.Run("failed to fetch from EN, tx result is not failed", func() { + // lookup should try each of the 2 ENs in fixedENIDs + suite.execClient.On("GetTransactionErrorMessagesByBlockID", mock.Anything, mock.Anything).Return(nil, + status.Error(codes.Unavailable, "")).Twice() + + // Setup mock that the transaction error message is not found in storage. + suite.txErrorMessages.On("ByBlockID", blockId). + Return(nil, storage.ErrNotFound).Once() + + // Setup mock that the transaction results exists and is not failed. + suite.transactionResults.On("ByBlockID", blockId). + Return([]flow.LightTransactionResult{ + { + TransactionID: unittest.IdentifierFixture(), + Failed: false, + ComputationUsed: 0, + }, + { + TransactionID: unittest.IdentifierFixture(), + Failed: false, + ComputationUsed: 0, + }, + }, nil).Once() + + // Perform the lookup and expect no error and an empty error messages. + errMsg, err := backend.LookupErrorMessagesByBlockID(context.Background(), blockId, block.Header.Height) + suite.Require().NoError(err) + suite.Require().Empty(errMsg) + suite.assertAllExpectations() + }) - params.TxResultsIndex = index.NewTransactionResultsIndex(index.NewReporter(), suite.transactionResults) - err := params.TxResultsIndex.Initialize(reporter) - suite.Require().NoError(err) + // Test case: failed to fetch from EN, but the transaction result is failed. + suite.Run("failed to fetch from EN, tx result is failed", func() { + failedResultsByBlockID := []flow.LightTransactionResult{ + { + TransactionID: unittest.IdentifierFixture(), + Failed: true, + ComputationUsed: 0, + }, + { + TransactionID: unittest.IdentifierFixture(), + Failed: true, + ComputationUsed: 0, + }, + } - backend, err := New(params) - suite.Require().NoError(err) + // lookup should try each of the 2 ENs in fixedENIDs + suite.execClient.On("GetTransactionErrorMessagesByBlockID", mock.Anything, mock.Anything).Return(nil, + status.Error(codes.Unavailable, "")).Twice() - // pretend the first transaction has been cached, but there are multiple failed txns so still a request has to be made. - backend.txErrorMessagesCache.Add(resultsByBlockID[0].TransactionID, "some error") + // Setup mock that the transaction error messages is not found in storage. + suite.txErrorMessages.On("ByBlockID", blockId). + Return(nil, storage.ErrNotFound).Once() - suite.execClient.On("GetTransactionErrorMessagesByBlockID", mock.Anything, mock.Anything).Return(nil, - status.Error(codes.Unavailable, "")).Twice() + // Setup mock that the transaction results exists and is failed. + suite.transactionResults.On("ByBlockID", blockId). + Return(failedResultsByBlockID, nil).Once() - errMsg, err := backend.LookupErrorMessagesByBlockID(context.Background(), blockId, block.Header.Height) - suite.Require().Error(err) - suite.Require().Equal(codes.Unavailable, status.Code(err)) - suite.Require().Empty(errMsg) + // Setup mock expected the transaction error messages after retrieving the failed result. + expectedTxErrorMessages := make(map[flow.Identifier]string) + for _, result := range failedResultsByBlockID { + if result.Failed { + expectedTxErrorMessages[result.TransactionID] = DefaultFailedErrorMessage + } + } - suite.assertAllExpectations() + // Perform the lookup and expect the failed error messages to be returned. + errMsg, err := backend.LookupErrorMessagesByBlockID(context.Background(), blockId, block.Header.Height) + suite.Require().NoError(err) + suite.Require().Len(errMsg, len(expectedTxErrorMessages)) + for txID, expectedMessage := range expectedTxErrorMessages { + actualMessage, ok := errMsg[txID] + suite.Require().True(ok) + suite.Assert().Equal(expectedMessage, actualMessage) + } + suite.assertAllExpectations() + }) } // TestGetSystemTransaction_HappyPath tests that GetSystemTransaction call returns system chunk transaction. @@ -851,13 +990,9 @@ func (suite *Suite) TestGetSystemTransactionResult_HappyPath() { On("ByBlockID", block.ID()). Return(flow.ExecutionReceiptList{receipt1}, nil) - // create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) - // the connection factory should be used to get the execution node client params := suite.defaultBackendParams() - params.ConnFactory = connFactory + params.ConnFactory = suite.setupConnectionFactory() exeEventReq := &execproto.GetTransactionsByBlockIDRequest{ BlockId: blockID[:], @@ -1046,13 +1181,9 @@ func (suite *Suite) TestGetSystemTransactionResult_FailedEncodingConversion() { On("ByBlockID", block.ID()). Return(flow.ExecutionReceiptList{receipt1}, nil) - // create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) - // the connection factory should be used to get the execution node client params := suite.defaultBackendParams() - params.ConnFactory = connFactory + params.ConnFactory = suite.setupConnectionFactory() exeEventReq := &execproto.GetTransactionsByBlockIDRequest{ BlockId: blockID[:], @@ -1175,10 +1306,6 @@ func (suite *Suite) TestTransactionResultFromStorage() { suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) suite.snapshot.On("Head", mock.Anything).Return(block.Header, nil) - // create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) - // create a mock index reporter reporter := syncmock.NewIndexReporter(suite.T()) reporter.On("LowestIndexedHeight").Return(block.Header.Height, nil) @@ -1191,7 +1318,7 @@ func (suite *Suite) TestTransactionResultFromStorage() { // Set up the backend parameters and the backend instance params := suite.defaultBackendParams() // the connection factory should be used to get the execution node client - params.ConnFactory = connFactory + params.ConnFactory = suite.setupConnectionFactory() params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() params.TxResultQueryMode = IndexQueryModeLocalOnly @@ -1266,10 +1393,6 @@ func (suite *Suite) TestTransactionByIndexFromStorage() { suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) suite.snapshot.On("Head", mock.Anything).Return(block.Header, nil) - // Create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) - // create a mock index reporter reporter := syncmock.NewIndexReporter(suite.T()) reporter.On("LowestIndexedHeight").Return(block.Header.Height, nil) @@ -1282,7 +1405,7 @@ func (suite *Suite) TestTransactionByIndexFromStorage() { // Set up the backend parameters and the backend instance params := suite.defaultBackendParams() // the connection factory should be used to get the execution node client - params.ConnFactory = connFactory + params.ConnFactory = suite.setupConnectionFactory() params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() params.TxResultQueryMode = IndexQueryModeLocalOnly @@ -1363,10 +1486,6 @@ func (suite *Suite) TestTransactionResultsByBlockIDFromStorage() { suite.snapshot.On("Identities", mock.Anything).Return(fixedENIDs, nil) suite.snapshot.On("Head", mock.Anything).Return(block.Header, nil) - // create a mock connection factory - connFactory := connectionmock.NewConnectionFactory(suite.T()) - connFactory.On("GetExecutionAPIClient", mock.Anything).Return(suite.execClient, &mockCloser{}, nil) - // create a mock index reporter reporter := syncmock.NewIndexReporter(suite.T()) reporter.On("LowestIndexedHeight").Return(block.Header.Height, nil) @@ -1379,7 +1498,7 @@ func (suite *Suite) TestTransactionResultsByBlockIDFromStorage() { // Set up the state and snapshot mocks and the backend instance params := suite.defaultBackendParams() // the connection factory should be used to get the execution node client - params.ConnFactory = connFactory + params.ConnFactory = suite.setupConnectionFactory() params.FixedExecutionNodeIDs = fixedENIDs.NodeIDs().Strings() params.EventsIndex = index.NewEventsIndex(indexReporter, suite.events) diff --git a/engine/access/rpc/backend/node_selector.go b/engine/access/rpc/backend/node_selector.go index c7d2ada5fb4..7d9ee12f64c 100644 --- a/engine/access/rpc/backend/node_selector.go +++ b/engine/access/rpc/backend/node_selector.go @@ -3,12 +3,10 @@ package backend import ( "fmt" + commonrpc "github.com/onflow/flow-go/engine/common/rpc" "github.com/onflow/flow-go/model/flow" ) -// maxNodesCnt is the maximum number of nodes that will be contacted to complete an API request. -const maxNodesCnt = 3 - // NodeSelector is an interface that represents the ability to select node identities that the access node is trying to reach. // It encapsulates the internal logic of node selection and provides a way to change implementations for different types // of nodes. Implementations of this interface should define the Next method, which returns the next node identity to be @@ -32,7 +30,7 @@ func (n *NodeSelectorFactory) SelectNodes(nodes flow.IdentitySkeletonList) (Node var err error // If the circuit breaker is disabled, the legacy logic should be used, which selects only a specified number of nodes. if !n.circuitBreakerEnabled { - nodes, err = nodes.Sample(maxNodesCnt) + nodes, err = nodes.Sample(commonrpc.MaxNodesCnt) if err != nil { return nil, fmt.Errorf("sampling failed: %w", err) } diff --git a/engine/access/rpc/backend/transactions_local_data_provider.go b/engine/access/rpc/backend/transactions_local_data_provider.go index f68fb35ed4b..921580452ec 100644 --- a/engine/access/rpc/backend/transactions_local_data_provider.go +++ b/engine/access/rpc/backend/transactions_local_data_provider.go @@ -31,18 +31,16 @@ type TransactionErrorMessage interface { // Expected errors during normal operation: // - InsufficientExecutionReceipts - found insufficient receipts for given block ID. // - status.Error - remote GRPC call to EN has failed. - LookupErrorMessageByTransactionID(ctx context.Context, blockID flow.Identifier, transactionID flow.Identifier) (string, error) + LookupErrorMessageByTransactionID(ctx context.Context, blockID flow.Identifier, height uint64, transactionID flow.Identifier) (string, error) // LookupErrorMessageByIndex is a function type for getting transaction error message by index. // Expected errors during normal operation: - // - status.Error[codes.NotFound] - transaction result for given block ID and tx index is not available. // - InsufficientExecutionReceipts - found insufficient receipts for given block ID. // - status.Error - remote GRPC call to EN has failed. LookupErrorMessageByIndex(ctx context.Context, blockID flow.Identifier, height uint64, index uint32) (string, error) // LookupErrorMessagesByBlockID is a function type for getting transaction error messages by block ID. // Expected errors during normal operation: - // - status.Error[codes.NotFound] - transaction results for given block ID are not available. // - InsufficientExecutionReceipts - found insufficient receipts for given block ID. // - status.Error - remote GRPC call to EN has failed. LookupErrorMessagesByBlockID(ctx context.Context, blockID flow.Identifier, height uint64) (map[flow.Identifier]string, error) @@ -84,7 +82,7 @@ func (t *TransactionsLocalDataProvider) GetTransactionResultFromStorage( var txErrorMessage string var txStatusCode uint = 0 if txResult.Failed { - txErrorMessage, err = t.txErrorMessages.LookupErrorMessageByTransactionID(ctx, blockID, transactionID) + txErrorMessage, err = t.txErrorMessages.LookupErrorMessageByTransactionID(ctx, blockID, block.Header.Height, transactionID) if err != nil { return nil, err } diff --git a/engine/common/rpc/utils.go b/engine/common/rpc/utils.go new file mode 100644 index 00000000000..60eceff5bb3 --- /dev/null +++ b/engine/common/rpc/utils.go @@ -0,0 +1,268 @@ +package rpc + +import ( + "context" + "fmt" + "time" + + "github.com/rs/zerolog" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/model/flow/filter" + "github.com/onflow/flow-go/state/protocol" + "github.com/onflow/flow-go/storage" +) + +// minExecutionNodesCnt is the minimum number of execution nodes expected to have sent the execution receipt for a block +const minExecutionNodesCnt = 2 + +// maxAttemptsForExecutionReceipt is the maximum number of attempts to find execution receipts for a given block ID +const maxAttemptsForExecutionReceipt = 3 + +// MaxNodesCnt is the maximum number of nodes that will be contacted to complete an API request. +const MaxNodesCnt = 3 + +func IdentifierList(ids []string) (flow.IdentifierList, error) { + idList := make(flow.IdentifierList, len(ids)) + for i, idStr := range ids { + id, err := flow.HexStringToIdentifier(idStr) + if err != nil { + return nil, fmt.Errorf("failed to convert node id string %s to Flow Identifier: %w", id, err) + } + idList[i] = id + } + return idList, nil +} + +// ExecutionNodesForBlockID returns upto maxNodesCnt number of randomly chosen execution node identities +// which have executed the given block ID. +// If no such execution node is found, an InsufficientExecutionReceipts error is returned. +func ExecutionNodesForBlockID( + ctx context.Context, + blockID flow.Identifier, + executionReceipts storage.ExecutionReceipts, + state protocol.State, + log zerolog.Logger, + preferredENIdentifiers flow.IdentifierList, + fixedENIdentifiers flow.IdentifierList, +) (flow.IdentitySkeletonList, error) { + var ( + executorIDs flow.IdentifierList + err error + ) + + // check if the block ID is of the root block. If it is then don't look for execution receipts since they + // will not be present for the root block. + rootBlock := state.Params().FinalizedRoot() + + if rootBlock.ID() == blockID { + executorIdentities, err := state.Final().Identities(filter.HasRole[flow.Identity](flow.RoleExecution)) + if err != nil { + return nil, fmt.Errorf("failed to retreive execution IDs for block ID %v: %w", blockID, err) + } + executorIDs = executorIdentities.NodeIDs() + } else { + // try to find at least minExecutionNodesCnt execution node ids from the execution receipts for the given blockID + for attempt := 0; attempt < maxAttemptsForExecutionReceipt; attempt++ { + executorIDs, err = findAllExecutionNodes(blockID, executionReceipts, log) + if err != nil { + return nil, err + } + + if len(executorIDs) >= minExecutionNodesCnt { + break + } + + // log the attempt + log.Debug().Int("attempt", attempt).Int("max_attempt", maxAttemptsForExecutionReceipt). + Int("execution_receipts_found", len(executorIDs)). + Str("block_id", blockID.String()). + Msg("insufficient execution receipts") + + // if one or less execution receipts may have been received then re-query + // in the hope that more might have been received by now + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(100 * time.Millisecond << time.Duration(attempt)): + // retry after an exponential backoff + } + } + + receiptCnt := len(executorIDs) + // if less than minExecutionNodesCnt execution receipts have been received so far, then return random ENs + if receiptCnt < minExecutionNodesCnt { + newExecutorIDs, err := state.AtBlockID(blockID).Identities(filter.HasRole[flow.Identity](flow.RoleExecution)) + if err != nil { + return nil, fmt.Errorf("failed to retreive execution IDs for block ID %v: %w", blockID, err) + } + executorIDs = newExecutorIDs.NodeIDs() + } + } + + // choose from the preferred or fixed execution nodes + subsetENs, err := chooseExecutionNodes(state, executorIDs, preferredENIdentifiers, fixedENIdentifiers) + if err != nil { + return nil, fmt.Errorf("failed to retreive execution IDs for block ID %v: %w", blockID, err) + } + + if len(subsetENs) == 0 { + return nil, fmt.Errorf("no matching execution node found for block ID %v", blockID) + } + + return subsetENs, nil +} + +// findAllExecutionNodes find all the execution nodes ids from the execution receipts that have been received for the +// given blockID +func findAllExecutionNodes( + blockID flow.Identifier, + executionReceipts storage.ExecutionReceipts, + log zerolog.Logger, +) (flow.IdentifierList, error) { + // lookup the receipt's storage with the block ID + allReceipts, err := executionReceipts.ByBlockID(blockID) + if err != nil { + return nil, fmt.Errorf("failed to retreive execution receipts for block ID %v: %w", blockID, err) + } + + executionResultMetaList := make(flow.ExecutionReceiptMetaList, 0, len(allReceipts)) + for _, r := range allReceipts { + executionResultMetaList = append(executionResultMetaList, r.Meta()) + } + executionResultGroupedMetaList := executionResultMetaList.GroupByResultID() + + // maximum number of matching receipts found so far for any execution result id + maxMatchedReceiptCnt := 0 + // execution result id key for the highest number of matching receipts in the identicalReceipts map + var maxMatchedReceiptResultID flow.Identifier + + // find the largest list of receipts which have the same result ID + for resultID, executionReceiptList := range executionResultGroupedMetaList { + currentMatchedReceiptCnt := executionReceiptList.Size() + if currentMatchedReceiptCnt > maxMatchedReceiptCnt { + maxMatchedReceiptCnt = currentMatchedReceiptCnt + maxMatchedReceiptResultID = resultID + } + } + + // if there are more than one execution result for the same block ID, log as error + if executionResultGroupedMetaList.NumberGroups() > 1 { + identicalReceiptsStr := fmt.Sprintf("%v", flow.GetIDs(allReceipts)) + log.Error(). + Str("block_id", blockID.String()). + Str("execution_receipts", identicalReceiptsStr). + Msg("execution receipt mismatch") + } + + // pick the largest list of matching receipts + matchingReceiptMetaList := executionResultGroupedMetaList.GetGroup(maxMatchedReceiptResultID) + + metaReceiptGroupedByExecutorID := matchingReceiptMetaList.GroupByExecutorID() + + // collect all unique execution node ids from the receipts + var executorIDs flow.IdentifierList + for executorID := range metaReceiptGroupedByExecutorID { + executorIDs = append(executorIDs, executorID) + } + + return executorIDs, nil +} + +// chooseExecutionNodes finds the subset of execution nodes defined in the identity table by first +// choosing the preferred execution nodes which have executed the transaction. If no such preferred +// execution nodes are found, then the fixed execution nodes defined in the identity table are returned +// If neither preferred nor fixed nodes are defined, then all execution node matching the executor IDs are returned. +// e.g. If execution nodes in identity table are {1,2,3,4}, preferred ENs are defined as {2,3,4} +// and the executor IDs is {1,2,3}, then {2, 3} is returned as the chosen subset of ENs +func chooseExecutionNodes( + state protocol.State, + executorIDs flow.IdentifierList, + preferredENIdentifiers flow.IdentifierList, + fixedENIdentifiers flow.IdentifierList, +) (flow.IdentitySkeletonList, error) { + allENs, err := state.Final().Identities(filter.HasRole[flow.Identity](flow.RoleExecution)) + if err != nil { + return nil, fmt.Errorf("failed to retrieve all execution IDs: %w", err) + } + + // choose from preferred EN IDs + if len(preferredENIdentifiers) > 0 { + chosenIDs := ChooseFromPreferredENIDs(allENs, executorIDs, preferredENIdentifiers) + return chosenIDs.ToSkeleton(), nil + } + + // if no preferred EN ID is found, then choose from the fixed EN IDs + if len(fixedENIdentifiers) > 0 { + // choose fixed ENs which have executed the transaction + chosenIDs := allENs.Filter(filter.And( + filter.HasNodeID[flow.Identity](fixedENIdentifiers...), + filter.HasNodeID[flow.Identity](executorIDs...), + )) + if len(chosenIDs) > 0 { + return chosenIDs.ToSkeleton(), nil + } + // if no such ENs are found, then just choose all fixed ENs + chosenIDs = allENs.Filter(filter.HasNodeID[flow.Identity](fixedENIdentifiers...)) + return chosenIDs.ToSkeleton(), nil + } + + // if no preferred or fixed ENs have been specified, then return all executor IDs i.e., no preference at all + return allENs.Filter(filter.HasNodeID[flow.Identity](executorIDs...)).ToSkeleton(), nil +} + +// ChooseFromPreferredENIDs finds the subset of execution nodes if preferred execution nodes are defined. +// If preferredENIdentifiers is set and there are less than maxNodesCnt nodes selected, than the list is padded up to +// maxNodesCnt nodes using the following order: +// 1. Use any EN with a receipt. +// 2. Use any preferred node not already selected. +// 3. Use any EN not already selected. +func ChooseFromPreferredENIDs(allENs flow.IdentityList, + executorIDs flow.IdentifierList, + preferredENIdentifiers flow.IdentifierList, +) flow.IdentityList { + var chosenIDs flow.IdentityList + + // filter for both preferred and executor IDs + chosenIDs = allENs.Filter(filter.And( + filter.HasNodeID[flow.Identity](preferredENIdentifiers...), + filter.HasNodeID[flow.Identity](executorIDs...), + )) + + if len(chosenIDs) >= MaxNodesCnt { + return chosenIDs + } + + // function to add nodes to chosenIDs if they are not already included + addIfNotExists := func(candidates flow.IdentityList) { + for _, en := range candidates { + _, exists := chosenIDs.ByNodeID(en.NodeID) + if !exists { + chosenIDs = append(chosenIDs, en) + if len(chosenIDs) >= MaxNodesCnt { + return + } + } + } + } + + // add any EN with a receipt + receiptENs := allENs.Filter(filter.HasNodeID[flow.Identity](executorIDs...)) + addIfNotExists(receiptENs) + if len(chosenIDs) >= MaxNodesCnt { + return chosenIDs + } + + // add any preferred node not already selected + preferredENs := allENs.Filter(filter.HasNodeID[flow.Identity](preferredENIdentifiers...)) + addIfNotExists(preferredENs) + if len(chosenIDs) >= MaxNodesCnt { + return chosenIDs + } + + // add any EN not already selected + addIfNotExists(allENs) + + return chosenIDs +} diff --git a/engine/execution/state/bootstrap/bootstrap_test.go b/engine/execution/state/bootstrap/bootstrap_test.go index 96457ebda93..5a9d394a5ac 100644 --- a/engine/execution/state/bootstrap/bootstrap_test.go +++ b/engine/execution/state/bootstrap/bootstrap_test.go @@ -53,7 +53,7 @@ func TestBootstrapLedger(t *testing.T) { } func TestBootstrapLedger_ZeroTokenSupply(t *testing.T) { - expectedStateCommitmentBytes, _ := hex.DecodeString("edc0ffd7e797e2383bc68b694645e6b62b0d996498bcff9b492bf4d426798ec5") + expectedStateCommitmentBytes, _ := hex.DecodeString("6e70a1ff40e4312a547d588a4355a538610bc22844a1faa907b4ec333ff1eca9") expectedStateCommitment, err := flow.ToStateCommitment(expectedStateCommitmentBytes) require.NoError(t, err) diff --git a/follower/follower_builder.go b/follower/follower_builder.go index 2988ee2767e..6d7b909125c 100644 --- a/follower/follower_builder.go +++ b/follower/follower_builder.go @@ -1,16 +1,12 @@ package follower import ( - "context" "encoding/json" "errors" "fmt" "strings" - dht "github.com/libp2p/go-libp2p-kad-dht" - "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/routing" "github.com/onflow/crypto" "github.com/rs/zerolog" @@ -44,17 +40,12 @@ import ( cborcodec "github.com/onflow/flow-go/network/codec/cbor" "github.com/onflow/flow-go/network/converter" "github.com/onflow/flow-go/network/p2p" - p2pbuilder "github.com/onflow/flow-go/network/p2p/builder" - p2pbuilderconfig "github.com/onflow/flow-go/network/p2p/builder/config" "github.com/onflow/flow-go/network/p2p/cache" "github.com/onflow/flow-go/network/p2p/conduit" - p2pdht "github.com/onflow/flow-go/network/p2p/dht" "github.com/onflow/flow-go/network/p2p/keyutils" p2plogging "github.com/onflow/flow-go/network/p2p/logging" - "github.com/onflow/flow-go/network/p2p/subscription" "github.com/onflow/flow-go/network/p2p/translator" "github.com/onflow/flow-go/network/p2p/unicast/protocols" - "github.com/onflow/flow-go/network/p2p/utils" "github.com/onflow/flow-go/network/slashing" "github.com/onflow/flow-go/network/underlay" "github.com/onflow/flow-go/network/validator" @@ -84,19 +75,14 @@ import ( // For a node running as a standalone process, the config fields will be populated from the command line params, // while for a node running as a library, the config fields are expected to be initialized by the caller. type FollowerServiceConfig struct { - bootstrapNodeAddresses []string - bootstrapNodePublicKeys []string - bootstrapIdentities flow.IdentitySkeletonList // the identity list of bootstrap peers the node uses to discover other nodes - NetworkKey crypto.PrivateKey // the networking key passed in by the caller when being used as a library - baseOptions []cmd.Option + bootstrapIdentities flow.IdentitySkeletonList // the identity list of bootstrap peers the node uses to discover other nodes + NetworkKey crypto.PrivateKey // the networking key passed in by the caller when being used as a library + baseOptions []cmd.Option } // DefaultFollowerServiceConfig defines all the default values for the FollowerServiceConfig func DefaultFollowerServiceConfig() *FollowerServiceConfig { - return &FollowerServiceConfig{ - bootstrapNodeAddresses: []string{}, - bootstrapNodePublicKeys: []string{}, - } + return &FollowerServiceConfig{} } // FollowerServiceBuilder provides the common functionality needed to bootstrap a Flow staked and observer @@ -136,7 +122,7 @@ func (builder *FollowerServiceBuilder) deriveBootstrapPeerIdentities() error { return nil } - ids, err := BootstrapIdentities(builder.bootstrapNodeAddresses, builder.bootstrapNodePublicKeys) + ids, err := builder.DeriveBootstrapPeerIdentities() if err != nil { return fmt.Errorf("failed to derive bootstrap peer identities: %w", err) } @@ -535,86 +521,15 @@ func (builder *FollowerServiceBuilder) validateParams() error { if len(builder.bootstrapIdentities) > 0 { return nil } - if len(builder.bootstrapNodeAddresses) == 0 { + if len(builder.BootstrapNodeAddresses) == 0 { return errors.New("no bootstrap node address provided") } - if len(builder.bootstrapNodeAddresses) != len(builder.bootstrapNodePublicKeys) { + if len(builder.BootstrapNodeAddresses) != len(builder.BootstrapNodePublicKeys) { return errors.New("number of bootstrap node addresses and public keys should match") } return nil } -// initPublicLibp2pNode creates a libp2p node for the follower service in public (unstaked) network. -// The LibP2P host is created with the following options: -// - DHT as client and seeded with the given bootstrap peers -// - The specified bind address as the listen address -// - The passed in private key as the libp2p key -// - No connection gater -// - No connection manager -// - No peer manager -// - Default libp2p pubsub options -// -// Args: -// - networkKey: the private key to use for the libp2p node -// -// Returns: -// - p2p.LibP2PNode: the libp2p node -// - error: if any error occurs. Any error returned from this function is irrecoverable. -func (builder *FollowerServiceBuilder) initPublicLibp2pNode(networkKey crypto.PrivateKey) (p2p.LibP2PNode, error) { - var pis []peer.AddrInfo - - for _, b := range builder.bootstrapIdentities { - pi, err := utils.PeerAddressInfo(*b) - if err != nil { - return nil, fmt.Errorf("could not extract peer address info from bootstrap identity %v: %w", b, err) - } - - pis = append(pis, pi) - } - - node, err := p2pbuilder.NewNodeBuilder( - builder.Logger, - &builder.FlowConfig.NetworkConfig.GossipSub, - &p2pbuilderconfig.MetricsConfig{ - HeroCacheFactory: builder.HeroCacheMetricsFactory(), - Metrics: builder.Metrics.Network, - }, - network.PublicNetwork, - builder.BaseConfig.BindAddr, - networkKey, - builder.SporkID, - builder.IdentityProvider, - &builder.FlowConfig.NetworkConfig.ResourceManager, - p2pbuilderconfig.PeerManagerDisableConfig(), // disable peer manager for follower - &p2p.DisallowListCacheConfig{ - MaxSize: builder.FlowConfig.NetworkConfig.DisallowListNotificationCacheSize, - Metrics: metrics.DisallowListCacheMetricsFactory(builder.HeroCacheMetricsFactory(), network.PublicNetwork), - }, - &p2pbuilderconfig.UnicastConfig{ - Unicast: builder.FlowConfig.NetworkConfig.Unicast, - }). - SetSubscriptionFilter( - subscription.NewRoleBasedFilter( - subscription.UnstakedRole, builder.IdentityProvider, - ), - ). - SetRoutingSystem(func(ctx context.Context, h host.Host) (routing.Routing, error) { - return p2pdht.NewDHT(ctx, h, protocols.FlowPublicDHTProtocolID(builder.SporkID), - builder.Logger, - builder.Metrics.Network, - p2pdht.AsClient(), - dht.BootstrapPeers(pis...), - ) - }).Build() - if err != nil { - return nil, fmt.Errorf("could not build public libp2p node: %w", err) - } - - builder.LibP2PNode = node - - return builder.LibP2PNode, nil -} - // initObserverLocal initializes the observer's ID, network key and network address // Currently, it reads a node-info.priv.json like any other node. // TODO: read the node ID from the special bootstrap files @@ -651,11 +566,12 @@ func (builder *FollowerServiceBuilder) enqueuePublicNetworkInit() { builder. Component("public libp2p node", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { var err error - publicLibp2pNode, err = builder.initPublicLibp2pNode(node.NetworkKey) + publicLibp2pNode, err = builder.BuildPublicLibp2pNode(builder.BaseConfig.BindAddr, builder.bootstrapIdentities) if err != nil { - return nil, fmt.Errorf("could not create public libp2p node: %w", err) + return nil, fmt.Errorf("could not build public libp2p node: %w", err) } + builder.LibP2PNode = publicLibp2pNode return publicLibp2pNode, nil }). Component("public network", func(node *cmd.NodeConfig) (module.ReadyDoneAware, error) { diff --git a/fvm/bootstrap.go b/fvm/bootstrap.go index 5c7e09d31ed..0f3526c8d83 100644 --- a/fvm/bootstrap.go +++ b/fvm/bootstrap.go @@ -351,6 +351,7 @@ func (b *bootstrapExecutor) Execute() error { b.deployViewResolver(service, &env) b.deployBurner(service, &env) + b.deployCrypto(service, &env) err = expectAccounts(1) if err != nil { @@ -523,6 +524,22 @@ func (b *bootstrapExecutor) deployBurner(deployTo flow.Address, env *templates.E panicOnMetaInvokeErrf("failed to deploy burner contract: %s", txError, err) } +func (b *bootstrapExecutor) deployCrypto(deployTo flow.Address, env *templates.Environment) { + contract := contracts.Crypto() + + txError, err := b.invokeMetaTransaction( + b.ctx, + Transaction( + blueprints.DeployContractTransaction( + deployTo, + contract, + "Crypto"), + 0), + ) + env.CryptoAddress = deployTo.String() + panicOnMetaInvokeErrf("failed to deploy crypto contract: %s", txError, err) +} + func (b *bootstrapExecutor) deployMetadataViews(fungibleToken, nonFungibleToken flow.Address, env *templates.Environment) { mvContract := contracts.MetadataViews(*env) diff --git a/fvm/crypto/hash_test.go b/fvm/crypto/hash_test.go index 29eeb7ec09c..6c8ba0354c8 100644 --- a/fvm/crypto/hash_test.go +++ b/fvm/crypto/hash_test.go @@ -2,10 +2,9 @@ package crypto_test import ( "crypto/rand" - "testing" - "crypto/sha256" "crypto/sha512" + "testing" "github.com/onflow/crypto/hash" "github.com/stretchr/testify/assert" diff --git a/fvm/environment/contract_reader.go b/fvm/environment/contract_reader.go index e3c868ee2e9..5c8aba5cc25 100644 --- a/fvm/environment/contract_reader.go +++ b/fvm/environment/contract_reader.go @@ -6,6 +6,7 @@ import ( "github.com/onflow/cadence/ast" "github.com/onflow/cadence/common" "github.com/onflow/cadence/runtime" + "github.com/onflow/cadence/stdlib" "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/tracing" @@ -15,21 +16,23 @@ import ( // ContractReader provide read access to contracts. type ContractReader struct { - tracer tracing.TracerSpan - meter Meter - - accounts Accounts + tracer tracing.TracerSpan + meter Meter + accounts Accounts + cryptoContractAddress common.Address } func NewContractReader( tracer tracing.TracerSpan, meter Meter, accounts Accounts, + cryptoContractAddress common.Address, ) *ContractReader { return &ContractReader{ - tracer: tracer, - meter: meter, - accounts: accounts, + tracer: tracer, + meter: meter, + accounts: accounts, + cryptoContractAddress: cryptoContractAddress, } } @@ -69,12 +72,37 @@ func (reader *ContractReader) ResolveLocation( return nil, fmt.Errorf("resolve location failed: %w", err) } + return ResolveLocation( + identifiers, + location, + reader.accounts.GetContractNames, + reader.cryptoContractAddress, + ) +} + +func ResolveLocation( + identifiers []ast.Identifier, + location common.Location, + getContractNames func(flow.Address) ([]string, error), + cryptoContractAddress common.Address, +) ([]runtime.ResolvedLocation, error) { + addressLocation, isAddress := location.(common.AddressLocation) // if the location is not an address location, e.g. an identifier location - // (`import Crypto`), then return a single resolved location which declares - // all identifiers. + // then return a single resolved location which declares all identifiers. if !isAddress { + + // if the location is the Crypto contract, + // translate it to the address of the Crypto contract on the chain + + if location == stdlib.CryptoContractLocation { + location = common.AddressLocation{ + Address: cryptoContractAddress, + Name: string(stdlib.CryptoContractLocation), + } + } + return []runtime.ResolvedLocation{ { Location: location, @@ -87,9 +115,13 @@ func (reader *ContractReader) ResolveLocation( // and no specific identifiers where requested in the import statement, // then fetch all identifiers at this address if len(identifiers) == 0 { + if getContractNames == nil { + return nil, fmt.Errorf("no identifiers provided") + } + address := flow.ConvertAddress(addressLocation.Address) - contractNames, err := reader.accounts.GetContractNames(address) + contractNames, err := getContractNames(address) if err != nil { return nil, fmt.Errorf("resolving location failed: %w", err) } diff --git a/fvm/environment/facade_env.go b/fvm/environment/facade_env.go index 7366dd4a89e..ab31db4e1d3 100644 --- a/fvm/environment/facade_env.go +++ b/fvm/environment/facade_env.go @@ -11,6 +11,7 @@ import ( "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/snapshot" "github.com/onflow/flow-go/fvm/storage/state" + "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/fvm/tracing" ) @@ -64,12 +65,15 @@ func newFacadeEnvironment( accounts := NewAccounts(txnState) logger := NewProgramLogger(tracer, params.ProgramLoggerParams) runtime := NewRuntime(params.RuntimeParams) + chain := params.Chain systemContracts := NewSystemContracts( - params.Chain, + chain, tracer, logger, runtime) + sc := systemcontracts.SystemContractsForChain(chain.ChainID()) + env := &facadeEnvironment{ Runtime: runtime, @@ -134,6 +138,7 @@ func newFacadeEnvironment( tracer, meter, accounts, + common.Address(sc.Crypto.Address), ), ContractUpdater: NoContractUpdater{}, Programs: NewPrograms( diff --git a/fvm/environment/meter.go b/fvm/environment/meter.go index 00a70df2d30..614cd124a52 100644 --- a/fvm/environment/meter.go +++ b/fvm/environment/meter.go @@ -4,6 +4,7 @@ import ( "context" "github.com/onflow/cadence/common" + "github.com/onflow/cadence/runtime" "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/meter" @@ -70,18 +71,13 @@ var MainnetExecutionEffortWeights = meter.ExecutionEffortWeights{ } type Meter interface { - MeterComputation(common.ComputationKind, uint) error - ComputationUsed() (uint64, error) + runtime.MeterInterface + ComputationIntensities() meter.MeteredComputationIntensities ComputationAvailable(common.ComputationKind, uint) bool - MeterMemory(usage common.MemoryUsage) error - MemoryUsed() (uint64, error) - MeterEmittedEvent(byteSize uint64) error TotalEmittedEventBytes() uint64 - - InteractionUsed() (uint64, error) } type meterImpl struct { @@ -112,6 +108,10 @@ func (meter *meterImpl) ComputationAvailable( return meter.txnState.ComputationAvailable(kind, intensity) } +func (meter *meterImpl) ComputationRemaining(kind common.ComputationKind) uint { + return meter.txnState.ComputationRemaining(kind) +} + func (meter *meterImpl) ComputationUsed() (uint64, error) { return meter.txnState.TotalComputationUsed(), nil } diff --git a/fvm/environment/mock/meter.go b/fvm/environment/mock/meter.go index 638d12a85e5..d73a61c5ff7 100644 --- a/fvm/environment/mock/meter.go +++ b/fvm/environment/mock/meter.go @@ -137,9 +137,9 @@ func (_m *Meter) MemoryUsed() (uint64, error) { return r0, r1 } -// MeterComputation provides a mock function with given fields: _a0, _a1 -func (_m *Meter) MeterComputation(_a0 common.ComputationKind, _a1 uint) error { - ret := _m.Called(_a0, _a1) +// MeterComputation provides a mock function with given fields: operationType, intensity +func (_m *Meter) MeterComputation(operationType common.ComputationKind, intensity uint) error { + ret := _m.Called(operationType, intensity) if len(ret) == 0 { panic("no return value specified for MeterComputation") @@ -147,7 +147,7 @@ func (_m *Meter) MeterComputation(_a0 common.ComputationKind, _a1 uint) error { var r0 error if rf, ok := ret.Get(0).(func(common.ComputationKind, uint) error); ok { - r0 = rf(_a0, _a1) + r0 = rf(operationType, intensity) } else { r0 = ret.Error(0) } diff --git a/fvm/evm/stdlib/checking.go b/fvm/evm/stdlib/checking.go index 03819737de1..81faed8c298 100644 --- a/fvm/evm/stdlib/checking.go +++ b/fvm/evm/stdlib/checking.go @@ -1,11 +1,11 @@ package stdlib import ( - "fmt" - "github.com/onflow/cadence/common" "github.com/onflow/cadence/interpreter" "github.com/onflow/cadence/runtime" + + "github.com/onflow/flow-go/fvm/environment" ) // checkingInterface is a runtime.Interface implementation @@ -13,68 +13,41 @@ import ( // It is not suitable for execution. type checkingInterface struct { runtime.EmptyRuntimeInterface - SystemContractCodes map[common.AddressLocation][]byte - Programs map[runtime.Location]*interpreter.Program + SystemContractCodes map[common.Location][]byte + Programs map[runtime.Location]*interpreter.Program + cryptoContractAddress common.Address } var _ runtime.Interface = &checkingInterface{} -func (*checkingInterface) ResolveLocation( +func (i *checkingInterface) ResolveLocation( identifiers []runtime.Identifier, location runtime.Location, ) ( []runtime.ResolvedLocation, error, ) { - - addressLocation, isAddress := location.(common.AddressLocation) - - // if the location is not an address location, e.g. an identifier location - // (`import Crypto`), then return a single resolved location which declares - // all identifiers. - if !isAddress { - return []runtime.ResolvedLocation{ - { - Location: location, - Identifiers: identifiers, - }, - }, nil - } - - if len(identifiers) == 0 { - return nil, fmt.Errorf("no identifiers provided") - } - - // return one resolved location per identifier. - // each resolved location is an address contract location - resolvedLocations := make([]runtime.ResolvedLocation, len(identifiers)) - for i := range resolvedLocations { - identifier := identifiers[i] - resolvedLocations[i] = runtime.ResolvedLocation{ - Location: common.AddressLocation{ - Address: addressLocation.Address, - Name: identifier.Identifier, - }, - Identifiers: []runtime.Identifier{identifier}, - } - } - - return resolvedLocations, nil + return environment.ResolveLocation( + identifiers, + location, + nil, + i.cryptoContractAddress, + ) } -func (r *checkingInterface) GetOrLoadProgram( +func (i *checkingInterface) GetOrLoadProgram( location runtime.Location, load func() (*interpreter.Program, error), ) ( program *interpreter.Program, err error, ) { - if r.Programs == nil { - r.Programs = map[runtime.Location]*interpreter.Program{} + if i.Programs == nil { + i.Programs = map[runtime.Location]*interpreter.Program{} } var ok bool - program, ok = r.Programs[location] + program, ok = i.Programs[location] if ok { return } @@ -84,11 +57,15 @@ func (r *checkingInterface) GetOrLoadProgram( // NOTE: important: still set empty program, // even if error occurred - r.Programs[location] = program + i.Programs[location] = program return } -func (r *checkingInterface) GetAccountContractCode(location common.AddressLocation) (code []byte, err error) { - return r.SystemContractCodes[location], nil +func (i *checkingInterface) GetCode(location common.Location) ([]byte, error) { + return i.SystemContractCodes[location], nil +} + +func (i *checkingInterface) GetAccountContractCode(location common.AddressLocation) (code []byte, err error) { + return i.SystemContractCodes[location], nil } diff --git a/fvm/evm/stdlib/contract.cdc b/fvm/evm/stdlib/contract.cdc index 986313c50df..a547768362e 100644 --- a/fvm/evm/stdlib/contract.cdc +++ b/fvm/evm/stdlib/contract.cdc @@ -74,7 +74,7 @@ contract EVM { /// This data helps to replay the transactions without the need to /// have access to the full cadence state data. precompiledCalls: [UInt8], - /// stateUpdateChecksum provides a mean to validate + /// stateUpdateChecksum provides a mean to validate /// the updates to the storage when re-executing a transaction off-chain. stateUpdateChecksum: [UInt8; 4] ) @@ -679,7 +679,6 @@ contract EVM { signatures: [[UInt8]], evmAddress: [UInt8; 20] ): ValidationResult { - // make signature set first // check number of signatures matches number of key indices if keyIndices.length != signatures.length { @@ -689,39 +688,58 @@ contract EVM { ) } - var signatureSet: [Crypto.KeyListSignature] = [] - for signatureIndex, signature in signatures{ - signatureSet.append(Crypto.KeyListSignature( - keyIndex: Int(keyIndices[signatureIndex]), - signature: signature - )) - } - // fetch account let acc = getAccount(address) - // constructing key list + var signatureSet: [Crypto.KeyListSignature] = [] let keyList = Crypto.KeyList() - for signature in signatureSet { - let keyRef = acc.keys.get(keyIndex: signature.keyIndex) - if keyRef == nil { - return ValidationResult( - isValid: false, - problem: "invalid key index" - ) - } - let key = keyRef! - if key.isRevoked { - return ValidationResult( - isValid: false, - problem: "account key is revoked" - ) + var keyListLength = 0 + let seenAccountKeyIndices: {Int: Int} = {} + for signatureIndex, signature in signatures{ + // index of the key on the account + let accountKeyIndex = Int(keyIndices[signatureIndex]!) + // index of the key in the key list + var keyListIndex = 0 + + if !seenAccountKeyIndices.containsKey(accountKeyIndex) { + // fetch account key with accountKeyIndex + if let key = acc.keys.get(keyIndex: accountKeyIndex) { + if key.isRevoked { + return ValidationResult( + isValid: false, + problem: "account key is revoked" + ) + } + + keyList.add( + key.publicKey, + hashAlgorithm: key.hashAlgorithm, + // normalization factor. We need to divide by 1000 because the + // `Crypto.KeyList.verify()` function expects the weight to be + // in the range [0, 1]. 1000 is the key weight threshold. + weight: key.weight / 1000.0, + ) + + keyListIndex = keyListLength + keyListLength = keyListLength + 1 + seenAccountKeyIndices[accountKeyIndex] = keyListIndex + } else { + return ValidationResult( + isValid: false, + problem: "invalid key index" + ) + } + } else { + // if we have already seen this accountKeyIndex, use the keyListIndex + // that was previously assigned to it + // `Crypto.KeyList.verify()` knows how to handle duplicate keys + keyListIndex = seenAccountKeyIndices[accountKeyIndex]! } - keyList.add( - key.publicKey, - hashAlgorithm: key.hashAlgorithm, - weight: key.weight, - ) + + signatureSet.append(Crypto.KeyListSignature( + keyIndex: keyListIndex, + signature: signature + )) } let isValid = keyList.verify( diff --git a/fvm/evm/stdlib/contract_test.go b/fvm/evm/stdlib/contract_test.go index 619e8d7a63f..9bc3092c9ec 100644 --- a/fvm/evm/stdlib/contract_test.go +++ b/fvm/evm/stdlib/contract_test.go @@ -1,6 +1,7 @@ package stdlib_test import ( + "bytes" "encoding/binary" "encoding/hex" "math/big" @@ -30,6 +31,26 @@ import ( "github.com/onflow/flow-go/model/flow" ) +func newLocationResolver( + cryptoContractAddress flow.Address, +) func( + identifiers []runtime.Identifier, + location runtime.Location, +) ([]runtime.ResolvedLocation, error) { + cryptoContractAddress2 := common.Address(cryptoContractAddress) + return func( + identifiers []runtime.Identifier, + location runtime.Location, + ) ([]runtime.ResolvedLocation, error) { + return environment.ResolveLocation( + identifiers, + location, + nil, + cryptoContractAddress2, + ) + } +} + type testContractHandler struct { flowTokenAddress common.Address evmContractAddress common.Address @@ -222,6 +243,7 @@ func deployContracts( NonFungibleTokenAddress: contractsAddressHex, MetadataViewsAddress: contractsAddressHex, FungibleTokenMetadataViewsAddress: contractsAddressHex, + CryptoAddress: contractsAddressHex, } contracts := []struct { @@ -229,6 +251,10 @@ func deployContracts( code []byte deployTx []byte }{ + { + name: "Crypto", + code: coreContracts.Crypto(), + }, { name: "ViewResolver", code: coreContracts.ViewResolver(), @@ -373,7 +399,7 @@ func TestEVMEncodeABI(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -474,7 +500,7 @@ func TestEVMEncodeABIByteTypes(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -921,7 +947,7 @@ func TestEVMEncodeABIBytesRoundtrip(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -1279,7 +1305,7 @@ func TestEVMEncodeABIComputation(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -1374,7 +1400,7 @@ func TestEVMEncodeABIComputationEmptyDynamicVariables(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -1478,7 +1504,7 @@ func TestEVMEncodeABIComputationDynamicVariablesAboveChunkSize(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -1576,7 +1602,7 @@ func TestEVMDecodeABI(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -1709,7 +1735,7 @@ func TestEVMDecodeABIComputation(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -1786,7 +1812,7 @@ func TestEVMEncodeDecodeABIRoundtripForUintIntTypes(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -2252,7 +2278,7 @@ func TestEVMEncodeDecodeABIRoundtrip(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -2330,7 +2356,7 @@ func TestEVMEncodeDecodeABIErrors(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -2416,7 +2442,7 @@ func TestEVMEncodeDecodeABIErrors(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -2501,7 +2527,7 @@ func TestEVMEncodeDecodeABIErrors(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -2587,7 +2613,7 @@ func TestEVMEncodeDecodeABIErrors(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -2673,7 +2699,7 @@ func TestEVMEncodeDecodeABIErrors(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -2769,7 +2795,7 @@ func TestEVMEncodeDecodeABIErrors(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -2855,7 +2881,7 @@ func TestEVMEncodeDecodeABIErrors(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -2941,7 +2967,7 @@ func TestEVMEncodeDecodeABIErrors(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -3027,7 +3053,7 @@ func TestEVMEncodeDecodeABIErrors(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -3113,7 +3139,7 @@ func TestEVMEncodeDecodeABIErrors(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -3199,7 +3225,7 @@ func TestEVMEncodeDecodeABIErrors(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -3315,7 +3341,7 @@ func TestEVMEncodeABIWithSignature(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -3448,7 +3474,7 @@ func TestEVMDecodeABIWithSignature(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -3568,7 +3594,7 @@ func TestEVMDecodeABIWithSignatureMismatch(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -3671,7 +3697,7 @@ func TestEVMAddressConstructionAndReturn(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -3773,7 +3799,7 @@ func TestEVMAddressSerializationAndDeserialization(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -3950,7 +3976,7 @@ func TestBalanceConstructionAndReturn(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -4080,7 +4106,7 @@ func TestEVMRun(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -4197,7 +4223,7 @@ func TestEVMDryRun(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -4336,7 +4362,7 @@ func TestEVMBatchRun(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -4436,7 +4462,7 @@ func TestEVMCreateCadenceOwnedAccount(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -4592,7 +4618,7 @@ func TestCadenceOwnedAccountCall(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -4711,7 +4737,7 @@ func TestEVMAddressDeposit(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -4824,7 +4850,7 @@ func TestCOADeposit(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -4995,7 +5021,7 @@ func TestCadenceOwnedAccountWithdraw(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -5148,7 +5174,7 @@ func TestCadenceOwnedAccountDeploy(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -5223,7 +5249,7 @@ func RunEVMScript( OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil @@ -5429,91 +5455,72 @@ func TestEVMAccountCodeHash(t *testing.T) { func TestEVMValidateCOAOwnershipProof(t *testing.T) { t.Parallel() - contractsAddress := flow.BytesToAddress([]byte{0x1}) - proof := &types.COAOwnershipProofInContext{ - COAOwnershipProof: types.COAOwnershipProof{ - Address: types.FlowAddress(contractsAddress), - CapabilityPath: "coa", - Signatures: []types.Signature{[]byte("signature")}, - KeyIndices: []uint64{0}, - }, - SignedData: []byte("signedData"), - EVMAddress: RandomAddress(t), - } - - handler := &testContractHandler{ - deployCOA: func(_ uint64) types.Address { - return proof.EVMAddress - }, - } - transactionEnvironment := newEVMTransactionEnvironment(handler, contractsAddress) - scriptEnvironment := newEVMScriptEnvironment(handler, contractsAddress) - - rt := runtime.NewInterpreterRuntime(runtime.Config{}) - - accountCodes := map[common.Location][]byte{} - var events []cadence.Event - - runtimeInterface := &TestRuntimeInterface{ - Storage: NewTestLedger(nil, nil), - OnGetSigningAccounts: func() ([]runtime.Address, error) { - return []runtime.Address{runtime.Address(contractsAddress)}, nil - }, - OnResolveLocation: LocationResolver, - OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { - accountCodes[location] = code - return nil - }, - OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { - code = accountCodes[location] - return code, nil - }, - OnEmitEvent: func(event cadence.Event) error { - events = append(events, event) - return nil - }, - OnDecodeArgument: func(b []byte, t cadence.Type) (cadence.Value, error) { - return json.Decode(nil, b) - }, - OnGetAccountKey: func(addr runtime.Address, index uint32) (*cadenceStdlib.AccountKey, error) { - require.Equal(t, proof.Address[:], addr[:]) - return &cadenceStdlib.AccountKey{ - PublicKey: &cadenceStdlib.PublicKey{}, - KeyIndex: index, - Weight: 100, - HashAlgo: sema.HashAlgorithmKECCAK_256, - IsRevoked: false, - }, nil - }, - OnVerifySignature: func( + validate := func( + proof *types.COAOwnershipProofInContext, + onGetAccountKey func(addr runtime.Address, index uint32) (*cadenceStdlib.AccountKey, error), + onVerifySignature func( signature []byte, tag string, sd, publicKey []byte, signatureAlgorithm runtime.SignatureAlgorithm, - hashAlgorithm runtime.HashAlgorithm) (bool, error) { - // require.Equal(t, []byte(signedData.ToGoValue()), st) - return true, nil - }, - } + hashAlgorithm runtime.HashAlgorithm) (bool, error), + ) (cadence.Value, error) { + handler := &testContractHandler{ + deployCOA: func(_ uint64) types.Address { + return proof.EVMAddress + }, + } + transactionEnvironment := newEVMTransactionEnvironment(handler, contractsAddress) + scriptEnvironment := newEVMScriptEnvironment(handler, contractsAddress) - nextTransactionLocation := NewTransactionLocationGenerator() - nextScriptLocation := NewScriptLocationGenerator() + rt := runtime.NewInterpreterRuntime(runtime.Config{}) - // Deploy contracts + accountCodes := map[common.Location][]byte{} + var events []cadence.Event - deployContracts( - t, - rt, - contractsAddress, - runtimeInterface, - transactionEnvironment, - nextTransactionLocation, - ) + runtimeInterface := &TestRuntimeInterface{ + Storage: NewTestLedger(nil, nil), + OnGetSigningAccounts: func() ([]runtime.Address, error) { + return []runtime.Address{runtime.Address(contractsAddress)}, nil + }, + OnResolveLocation: newLocationResolver(contractsAddress), + OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { + accountCodes[location] = code + return nil + }, + OnGetAccountContractCode: func(location common.AddressLocation) (code []byte, err error) { + code = accountCodes[location] + return code, nil + }, + OnEmitEvent: func(event cadence.Event) error { + events = append(events, event) + return nil + }, + OnDecodeArgument: func(b []byte, t cadence.Type) (cadence.Value, error) { + return json.Decode(nil, b) + }, + OnGetAccountKey: onGetAccountKey, + OnVerifySignature: onVerifySignature, + } + + nextTransactionLocation := NewTransactionLocationGenerator() + nextScriptLocation := NewScriptLocationGenerator() - setupTx := []byte(` + // Deploy contracts + + deployContracts( + t, + rt, + contractsAddress, + runtimeInterface, + transactionEnvironment, + nextTransactionLocation, + ) + + setupTx := []byte(` import EVM from 0x1 transaction { @@ -5531,55 +5538,211 @@ func TestEVMValidateCOAOwnershipProof(t *testing.T) { } }`) - err := rt.ExecuteTransaction( - runtime.Script{ - Source: setupTx, - }, - runtime.Context{ - Interface: runtimeInterface, - Environment: transactionEnvironment, - Location: nextTransactionLocation(), - }, - ) - require.NoError(t, err) + err := rt.ExecuteTransaction( + runtime.Script{ + Source: setupTx, + }, + runtime.Context{ + Interface: runtimeInterface, + Environment: transactionEnvironment, + Location: nextTransactionLocation(), + }, + ) + require.NoError(t, err) - script := []byte(` - import EVM from 0x1 + script := []byte(` + import EVM from 0x1 + + access(all) + fun main( + address: Address, + path: PublicPath, + signedData: [UInt8], + keyIndices: [UInt64], + signatures: [[UInt8]], + evmAddress: [UInt8; 20] + ): EVM.ValidationResult { + return EVM.validateCOAOwnershipProof( + address: address, + path: path, + signedData: signedData, + keyIndices: keyIndices, + signatures: signatures, + evmAddress: evmAddress + ) + } + `) - access(all) - fun main( - address: Address, - path: PublicPath, - signedData: [UInt8], - keyIndices: [UInt64], - signatures: [[UInt8]], - evmAddress: [UInt8; 20] - - ) { - EVM.validateCOAOwnershipProof( - address: address, - path: path, - signedData: signedData, - keyIndices: keyIndices, - signatures: signatures, - evmAddress: evmAddress - ) - } - `) + // Run script + result, err := rt.ExecuteScript( + runtime.Script{ + Source: script, + Arguments: EncodeArgs(proof.ToCadenceValues()), + }, + runtime.Context{ + Interface: runtimeInterface, + Environment: scriptEnvironment, + Location: nextScriptLocation(), + }, + ) - // Run script - _, err = rt.ExecuteScript( - runtime.Script{ - Source: script, - Arguments: EncodeArgs(proof.ToCadenceValues()), - }, - runtime.Context{ - Interface: runtimeInterface, - Environment: scriptEnvironment, - Location: nextScriptLocation(), - }, - ) - require.NoError(t, err) + return result, err + } + + t.Run("Single key", func(t *testing.T) { + proof := &types.COAOwnershipProofInContext{ + COAOwnershipProof: types.COAOwnershipProof{ + Address: types.FlowAddress(contractsAddress), + CapabilityPath: "coa", + Signatures: []types.Signature{[]byte("signature")}, + KeyIndices: []uint64{0}, + }, + SignedData: []byte("signedData"), + EVMAddress: RandomAddress(t), + } + + result, err := validate( + proof, + func( + addr runtime.Address, + index uint32, + ) (*cadenceStdlib.AccountKey, error) { + require.Equal(t, proof.Address[:], addr[:]) + return &cadenceStdlib.AccountKey{ + PublicKey: &cadenceStdlib.PublicKey{}, + KeyIndex: index, + Weight: 1000, + HashAlgo: sema.HashAlgorithmKECCAK_256, + IsRevoked: false, + }, nil + }, + func( + signature []byte, + tag string, + sd, + publicKey []byte, + signatureAlgorithm runtime.SignatureAlgorithm, + hashAlgorithm runtime.HashAlgorithm, + ) (bool, error) { + return true, nil + }, + ) + + require.NoError(t, err) + + isValid := result.(cadence.Struct).SearchFieldByName("isValid").(cadence.Bool) + require.True(t, bool(isValid)) + }) + + t.Run("Two keys", func(t *testing.T) { + proof := &types.COAOwnershipProofInContext{ + COAOwnershipProof: types.COAOwnershipProof{ + Address: types.FlowAddress(contractsAddress), + CapabilityPath: "coa", + Signatures: []types.Signature{[]byte("signature2"), []byte("signature0")}, + KeyIndices: []uint64{2, 0}, + }, + SignedData: []byte("signedData"), + EVMAddress: RandomAddress(t), + } + + result, err := validate( + proof, + func(addr runtime.Address, index uint32) (*cadenceStdlib.AccountKey, error) { + require.Equal(t, proof.Address[:], addr[:]) + return &cadenceStdlib.AccountKey{ + PublicKey: &cadenceStdlib.PublicKey{ + // encode the key index into the public key + PublicKey: []byte{byte(index)}, + }, + KeyIndex: index, + Weight: 1000, + HashAlgo: sema.HashAlgorithmKECCAK_256, + IsRevoked: false, + }, nil + }, + func( + signature []byte, + tag string, + sd, + publicKey []byte, + signatureAlgorithm runtime.SignatureAlgorithm, + hashAlgorithm runtime.HashAlgorithm, + ) (bool, error) { + if bytes.Equal(signature, []byte("signature2")) { + require.Equal(t, byte(2), publicKey[0]) + return true, nil + } else if bytes.Equal(signature, []byte("signature0")) { + require.Equal(t, byte(0), publicKey[0]) + return true, nil + } else { + return false, nil + } + }, + ) + + require.NoError(t, err) + + isValid := result.(cadence.Struct).SearchFieldByName("isValid").(cadence.Bool) + require.True(t, bool(isValid)) + }) + + t.Run("Two keys insufficient weight", func(t *testing.T) { + proof := &types.COAOwnershipProofInContext{ + COAOwnershipProof: types.COAOwnershipProof{ + Address: types.FlowAddress(contractsAddress), + CapabilityPath: "coa", + Signatures: []types.Signature{[]byte("signature2"), []byte("signature0")}, + KeyIndices: []uint64{2, 0}, + }, + SignedData: []byte("signedData"), + EVMAddress: RandomAddress(t), + } + + result, err := validate( + proof, + func(addr runtime.Address, index uint32) (*cadenceStdlib.AccountKey, error) { + require.Equal(t, proof.Address[:], addr[:]) + return &cadenceStdlib.AccountKey{ + PublicKey: &cadenceStdlib.PublicKey{ + // encode the key index into the public key + PublicKey: []byte{byte(index)}, + }, + KeyIndex: index, + Weight: 499, + HashAlgo: sema.HashAlgorithmKECCAK_256, + IsRevoked: false, + }, nil + }, + func( + signature []byte, + tag string, + sd, + publicKey []byte, + signatureAlgorithm runtime.SignatureAlgorithm, + hashAlgorithm runtime.HashAlgorithm, + ) (bool, error) { + if bytes.Equal(signature, []byte("signature2")) { + require.Equal(t, byte(2), publicKey[0]) + return true, nil + } else if bytes.Equal(signature, []byte("signature0")) { + require.Equal(t, byte(0), publicKey[0]) + return true, nil + } else { + return false, nil + } + }, + ) + + require.NoError(t, err) + + isValid := result.(cadence.Struct).SearchFieldByName("isValid").(cadence.Bool) + require.False(t, bool(isValid)) + message := result.(cadence.Struct). + SearchFieldByName("problem").(cadence.Optional). + Value.(cadence.String).String() + require.Equal(t, "\"the given signatures are not valid or provide enough weight\"", message) + }) } func TestInternalEVMAccess(t *testing.T) { @@ -5609,7 +5772,7 @@ func TestInternalEVMAccess(t *testing.T) { OnGetSigningAccounts: func() ([]runtime.Address, error) { return []runtime.Address{runtime.Address(contractsAddress)}, nil }, - OnResolveLocation: LocationResolver, + OnResolveLocation: newLocationResolver(contractsAddress), OnUpdateAccountContractCode: func(location common.AddressLocation, code []byte) error { accountCodes[location] = code return nil diff --git a/fvm/evm/stdlib/type.go b/fvm/evm/stdlib/type.go index ed3c1825f16..5ad44909bb4 100644 --- a/fvm/evm/stdlib/type.go +++ b/fvm/evm/stdlib/type.go @@ -32,8 +32,11 @@ func newContractType(chainID flow.ChainID) *sema.CompositeType { templatesEnv := contracts.AsTemplateEnv() + cryptoContractLocation := contracts.Crypto.Location() + runtimeInterface := &checkingInterface{ - SystemContractCodes: map[common.AddressLocation][]byte{ + cryptoContractAddress: cryptoContractLocation.Address, + SystemContractCodes: map[common.Location][]byte{ contracts.ViewResolver.Location(): coreContracts.ViewResolver(), contracts.Burner.Location(): coreContracts.Burner(), contracts.FungibleToken.Location(): coreContracts.FungibleToken(templatesEnv), @@ -41,6 +44,7 @@ func newContractType(chainID flow.ChainID) *sema.CompositeType { contracts.MetadataViews.Location(): coreContracts.MetadataViews(templatesEnv), contracts.FlowToken.Location(): coreContracts.FlowToken(templatesEnv), contracts.FungibleTokenMetadataViews.Location(): coreContracts.FungibleTokenMetadataViews(templatesEnv), + cryptoContractLocation: coreContracts.Crypto(), }, } diff --git a/fvm/evm/testutils/cadence.go b/fvm/evm/testutils/cadence.go index 3af633e74d7..12f4889a428 100644 --- a/fvm/evm/testutils/cadence.go +++ b/fvm/evm/testutils/cadence.go @@ -5,64 +5,10 @@ import ( "testing" "github.com/onflow/cadence" - "github.com/onflow/cadence/ast" - "github.com/onflow/cadence/common" "github.com/onflow/cadence/encoding/json" - "github.com/onflow/cadence/runtime" - "github.com/onflow/cadence/sema" "github.com/stretchr/testify/require" ) -// LocationResolver is a location Cadence runtime interface location resolver -// very similar to ContractReader.ResolveLocation, -// but it does not look up available contract names -func LocationResolver( - identifiers []ast.Identifier, - location common.Location, -) ( - result []sema.ResolvedLocation, - err error, -) { - addressLocation, isAddress := location.(common.AddressLocation) - - // if the location is not an address location, e.g. an identifier location - // (`import Crypto`), then return a single resolved location which declares - // all identifiers. - if !isAddress { - return []runtime.ResolvedLocation{ - { - Location: location, - Identifiers: identifiers, - }, - }, nil - } - - // if the location is an address, - // and no specific identifiers where requested in the import statement, - // then assume the imported identifier is the address location's identifier (the contract) - if len(identifiers) == 0 { - identifiers = []ast.Identifier{ - {Identifier: addressLocation.Name}, - } - } - - // return one resolved location per identifier. - // each resolved location is an address contract location - resolvedLocations := make([]runtime.ResolvedLocation, len(identifiers)) - for i := range resolvedLocations { - identifier := identifiers[i] - resolvedLocations[i] = runtime.ResolvedLocation{ - Location: common.AddressLocation{ - Address: addressLocation.Address, - Name: identifier.Identifier, - }, - Identifiers: []runtime.Identifier{identifier}, - } - } - - return resolvedLocations, nil -} - func EncodeArgs(argValues []cadence.Value) [][]byte { args := make([][]byte, len(argValues)) for i, arg := range argValues { diff --git a/fvm/fvm_bench_test.go b/fvm/fvm_bench_test.go index 33ff347a6f1..566cf81653a 100644 --- a/fvm/fvm_bench_test.go +++ b/fvm/fvm_bench_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - flow2 "github.com/onflow/flow-go-sdk" + flowsdk "github.com/onflow/flow-go-sdk" "github.com/onflow/flow-go-sdk/templates" "github.com/onflow/flow-go/engine/execution" @@ -315,13 +315,13 @@ func (b *BasicBlockExecutor) SetupAccounts(tb testing.TB, privateKeys []flow.Acc serviceAddress := b.Chain(tb).ServiceAddress() for _, privateKey := range privateKeys { - accountKey := flow2.NewAccountKey(). + accountKey := flowsdk.NewAccountKey(). FromPrivateKey(privateKey.PrivateKey). SetWeight(fvm.AccountKeyWeightThreshold). SetHashAlgo(privateKey.HashAlgo). SetSigAlgo(privateKey.SignAlgo) - sdkTX, err := templates.CreateAccount([]*flow2.AccountKey{accountKey}, []templates.Contract{}, flow2.BytesToAddress(serviceAddress.Bytes())) + sdkTX, err := templates.CreateAccount([]*flowsdk.AccountKey{accountKey}, []templates.Contract{}, flowsdk.BytesToAddress(serviceAddress.Bytes())) require.NoError(tb, err) txBody := flow.NewTransactionBody(). diff --git a/fvm/fvm_test.go b/fvm/fvm_test.go index 55d4d44bb92..0caabd49cb2 100644 --- a/fvm/fvm_test.go +++ b/fvm/fvm_test.go @@ -18,7 +18,10 @@ import ( "github.com/onflow/flow-go/fvm/storage" "github.com/onflow/flow-go/fvm/storage/state" - stdlib2 "github.com/onflow/cadence/stdlib" + cadenceStdlib "github.com/onflow/cadence/stdlib" + + flowsdk "github.com/onflow/flow-go-sdk" + "github.com/onflow/flow-go-sdk/test" envMock "github.com/onflow/flow-go/fvm/environment/mock" "github.com/onflow/flow-go/fvm/evm/events" @@ -980,7 +983,7 @@ func TestTransactionFeeDeduction(t *testing.T) { address := flow.ConvertAddress( cadence.SearchFieldByName( data.(cadence.Event), - stdlib2.AccountEventAddressParameter.Identifier, + cadenceStdlib.AccountEventAddressParameter.Identifier, ).(cadence.Address), ) @@ -1537,17 +1540,17 @@ func TestSettingExecutionWeights(t *testing.T) { SetScript([]byte(fmt.Sprintf(` import FungibleToken from 0x%s import FlowToken from 0x%s - + transaction() { let sentVault: @{FungibleToken.Vault} - + prepare(signer: auth(BorrowValue) &Account) { let vaultRef = signer.storage.borrow(from: /storage/flowTokenVault) ?? panic("Could not borrow reference to the owner's Vault!") - + self.sentVault <- vaultRef.withdraw(amount: 5.0) } - + execute { let recipient1 = getAccount(%s) let recipient2 = getAccount(%s) @@ -1565,7 +1568,7 @@ func TestSettingExecutionWeights(t *testing.T) { ?? panic("Could not borrow receiver reference to the recipient's Vault") let receiverRef5 = recipient5.capabilities.borrow<&{FungibleToken.Receiver}>(/public/flowTokenReceiver) ?? panic("Could not borrow receiver reference to the recipient's Vault") - + receiverRef1.deposit(from: <-self.sentVault.withdraw(amount: 1.0)) receiverRef2.deposit(from: <-self.sentVault.withdraw(amount: 1.0)) receiverRef3.deposit(from: <-self.sentVault.withdraw(amount: 1.0)) @@ -2341,7 +2344,7 @@ func TestInteractionLimit(t *testing.T) { address = flow.ConvertAddress( cadence.SearchFieldByName( data.(cadence.Event), - stdlib2.AccountEventAddressParameter.Identifier, + cadenceStdlib.AccountEventAddressParameter.Identifier, ).(cadence.Address), ) @@ -2925,7 +2928,7 @@ func TestEVM(t *testing.T) { sc := systemcontracts.SystemContractsForChain(chain.ChainID()) script := fvm.Script([]byte(fmt.Sprintf(` import EVM from %s - + access(all) fun main() { let bal = EVM.Balance(attoflow: 1000000000000000000) let acc <- EVM.createCadenceOwnedAccount() @@ -2988,7 +2991,7 @@ func TestEVM(t *testing.T) { script := fvm.Script([]byte(fmt.Sprintf(` import EVM from %s - + access(all) fun main() { destroy <- EVM.createCadenceOwnedAccount() @@ -3020,7 +3023,7 @@ func TestEVM(t *testing.T) { txBody := flow.NewTransactionBody(). SetScript([]byte(fmt.Sprintf(` import FungibleToken from %s - import FlowToken from %s + import FlowToken from %s import EVM from %s transaction() { @@ -3038,7 +3041,7 @@ func TestEVM(t *testing.T) { acc.deposit(from: <- amount) destroy acc - // commit blocks + // commit blocks evmHeartbeat.heartbeat() } }`, @@ -3239,6 +3242,178 @@ func TestAccountCapabilitiesPublishEntitledRejection(t *testing.T) { ) } +func TestCrypto(t *testing.T) { + t.Parallel() + + const chainID = flow.Testnet + + test := func(t *testing.T, importDecl string) { + + chain, vm := createChainAndVm(chainID) + + ctx := fvm.NewContext( + fvm.WithChain(chain), + fvm.WithCadenceLogging(true), + ) + + script := []byte(fmt.Sprintf( + ` + %s + + access(all) + fun main( + rawPublicKeys: [String], + weights: [UFix64], + domainSeparationTag: String, + signatures: [String], + toAddress: Address, + fromAddress: Address, + amount: UFix64 + ): Bool { + let keyList = Crypto.KeyList() + + var i = 0 + for rawPublicKey in rawPublicKeys { + keyList.add( + PublicKey( + publicKey: rawPublicKey.decodeHex(), + signatureAlgorithm: SignatureAlgorithm.ECDSA_P256 + ), + hashAlgorithm: HashAlgorithm.SHA3_256, + weight: weights[i], + ) + i = i + 1 + } + + let signatureSet: [Crypto.KeyListSignature] = [] + + var j = 0 + for signature in signatures { + signatureSet.append( + Crypto.KeyListSignature( + keyIndex: j, + signature: signature.decodeHex() + ) + ) + j = j + 1 + } + + // assemble the same message in cadence + let message = toAddress.toBytes() + .concat(fromAddress.toBytes()) + .concat(amount.toBigEndianBytes()) + + return keyList.verify( + signatureSet: signatureSet, + signedData: message, + domainSeparationTag: domainSeparationTag + ) + } + `, + importDecl, + )) + + accountKeys := test.AccountKeyGenerator() + + // create the keys + keyAlice, signerAlice := accountKeys.NewWithSigner() + keyBob, signerBob := accountKeys.NewWithSigner() + + // create the message that will be signed + addresses := test.AddressGenerator() + + toAddress := cadence.Address(addresses.New()) + fromAddress := cadence.Address(addresses.New()) + + amount, err := cadence.NewUFix64("100.00") + require.NoError(t, err) + + var message []byte + message = append(message, toAddress.Bytes()...) + message = append(message, fromAddress.Bytes()...) + message = append(message, amount.ToBigEndianBytes()...) + + // sign the message with Alice and Bob + signatureAlice, err := flowsdk.SignUserMessage(signerAlice, message) + require.NoError(t, err) + + signatureBob, err := flowsdk.SignUserMessage(signerBob, message) + require.NoError(t, err) + + publicKeys := cadence.NewArray([]cadence.Value{ + cadence.String(hex.EncodeToString(keyAlice.PublicKey.Encode())), + cadence.String(hex.EncodeToString(keyBob.PublicKey.Encode())), + }) + + // each signature has half weight + weightAlice, err := cadence.NewUFix64("0.5") + require.NoError(t, err) + + weightBob, err := cadence.NewUFix64("0.5") + require.NoError(t, err) + + weights := cadence.NewArray([]cadence.Value{ + weightAlice, + weightBob, + }) + + signatures := cadence.NewArray([]cadence.Value{ + cadence.String(hex.EncodeToString(signatureAlice)), + cadence.String(hex.EncodeToString(signatureBob)), + }) + + domainSeparationTag := cadence.String("FLOW-V0.0-user") + + arguments := []cadence.Value{ + publicKeys, + weights, + domainSeparationTag, + signatures, + toAddress, + fromAddress, + amount, + } + + encodedArguments := make([][]byte, 0, len(arguments)) + for _, argument := range arguments { + encodedArguments = append(encodedArguments, jsoncdc.MustEncode(argument)) + } + + snapshotTree := testutil.RootBootstrappedLedger(vm, ctx) + + _, output, err := vm.Run( + ctx, + fvm.Script(script). + WithArguments(encodedArguments...), + snapshotTree) + require.NoError(t, err) + + require.NoError(t, output.Err) + + result := output.Value + + assert.Equal(t, + cadence.NewBool(true), + result, + ) + } + + t.Run("identifier location", func(t *testing.T) { + t.Parallel() + + test(t, "import Crypto") + }) + + t.Run("address location", func(t *testing.T) { + t.Parallel() + + sc := systemcontracts.SystemContractsForChain(chainID) + cryptoContractAddress := sc.Crypto.Address.HexWithPrefix() + + test(t, fmt.Sprintf("import Crypto from %s", cryptoContractAddress)) + }) +} + func Test_MinimumRequiredVersion(t *testing.T) { chain := flow.Emulator.Chain() diff --git a/fvm/meter/computation_meter.go b/fvm/meter/computation_meter.go index 25e0b63ff54..d6f7fe55331 100644 --- a/fvm/meter/computation_meter.go +++ b/fvm/meter/computation_meter.go @@ -116,10 +116,28 @@ func (m *ComputationMeter) ComputationAvailable( if !ok { return true } + potentialComputationUsage := m.computationUsed + w*uint64(intensity) return potentialComputationUsage <= m.params.computationLimit } +// ComputationRemaining returns the remaining computation (intensity) left in the transaction for the given type +func (m *ComputationMeter) ComputationRemaining(kind common.ComputationKind) uint { + w, ok := m.params.computationWeights[kind] + // if not found return has capacity + // given the behaviour of MeterComputation is ignoring intensities without a set weight + if !ok { + return math.MaxUint + } + + remainingComputationUsage := m.params.computationLimit - m.computationUsed + if remainingComputationUsage <= 0 { + return 0 + } + + return uint(remainingComputationUsage / w) +} + // ComputationIntensities returns all the measured computational intensities func (m *ComputationMeter) ComputationIntensities() MeteredComputationIntensities { return m.computationIntensities diff --git a/fvm/meter/meter_test.go b/fvm/meter/meter_test.go index cba3643e151..f691b03b6d0 100644 --- a/fvm/meter/meter_test.go +++ b/fvm/meter/meter_test.go @@ -139,6 +139,27 @@ func TestWeightedComputationMetering(t *testing.T) { require.True(t, m.ComputationAvailable(1, 10)) }) + t.Run("check computation remaining", func(t *testing.T) { + m := meter.NewMeter( + meter.DefaultParameters(). + WithComputationLimit(10). + WithComputationWeights( + map[common.ComputationKind]uint64{0: 1 << meter.MeterExecutionInternalPrecisionBytes}), + ) + + remaining := m.ComputationRemaining(0) + require.Equal(t, uint(10), remaining) + + err := m.MeterComputation(0, 1) + require.NoError(t, err) + require.Equal(t, uint64(1), m.TotalComputationUsed()) + + require.Equal(t, uint(9), m.ComputationRemaining(0)) + + // test a type without a weight (default zero) + require.Equal(t, uint(math.MaxUint), m.ComputationRemaining(1)) + }) + t.Run("merge meters", func(t *testing.T) { compKind := common.ComputationKind(0) m := meter.NewMeter( diff --git a/fvm/storage/state/execution_state.go b/fvm/storage/state/execution_state.go index c9bde5d4190..12968e4b0a6 100644 --- a/fvm/storage/state/execution_state.go +++ b/fvm/storage/state/execution_state.go @@ -2,6 +2,7 @@ package state import ( "fmt" + "math" "github.com/coreos/go-semver/semver" @@ -244,6 +245,20 @@ func (state *ExecutionState) ComputationAvailable(kind common.ComputationKind, i return true } +// ComputationRemaining returns the available computation capacity without metering +func (state *ExecutionState) ComputationRemaining(kind common.ComputationKind) uint { + if state.finalized { + // if state is finalized return 0 + return 0 + } + + if state.enforceLimits { + return state.meter.ComputationRemaining(kind) + } + + return math.MaxUint +} + // TotalComputationUsed returns total computation used func (state *ExecutionState) TotalComputationUsed() uint64 { return state.meter.TotalComputationUsed() diff --git a/fvm/storage/state/transaction_state.go b/fvm/storage/state/transaction_state.go index 00f2ecce448..1b00bf78973 100644 --- a/fvm/storage/state/transaction_state.go +++ b/fvm/storage/state/transaction_state.go @@ -22,6 +22,7 @@ func (id NestedTransactionId) StateForTestingOnly() *ExecutionState { type Meter interface { MeterComputation(kind common.ComputationKind, intensity uint) error ComputationAvailable(kind common.ComputationKind, intensity uint) bool + ComputationRemaining(kind common.ComputationKind) uint ComputationIntensities() meter.MeteredComputationIntensities TotalComputationLimit() uint TotalComputationUsed() uint64 @@ -458,6 +459,10 @@ func (txnState *transactionState) ComputationAvailable( return txnState.current().ComputationAvailable(kind, intensity) } +func (txnState *transactionState) ComputationRemaining(kind common.ComputationKind) uint { + return txnState.current().ComputationRemaining(kind) +} + func (txnState *transactionState) MeterMemory( kind common.MemoryKind, intensity uint, diff --git a/fvm/systemcontracts/system_contracts.go b/fvm/systemcontracts/system_contracts.go index c483009ce83..ad9f66c4a65 100644 --- a/fvm/systemcontracts/system_contracts.go +++ b/fvm/systemcontracts/system_contracts.go @@ -43,6 +43,7 @@ const ( ContractNameViewResolver = "ViewResolver" ContractNameEVM = "EVM" ContractNameBurner = "Burner" + ContractNameCrypto = "Crypto" // AccountNameEVMStorage is not a contract, but a special account that is used to store EVM state AccountNameEVMStorage = "EVMStorageAccount" @@ -172,6 +173,7 @@ type SystemContracts struct { // Utility contracts Burner SystemContract + Crypto SystemContract } // AsTemplateEnv returns a template environment with all system contracts filled in. @@ -199,6 +201,7 @@ func (c SystemContracts) AsTemplateEnv() templates.Environment { ViewResolverAddress: c.ViewResolver.Address.Hex(), BurnerAddress: c.Burner.Address.Hex(), + CryptoAddress: c.Crypto.Address.Hex(), } } @@ -229,6 +232,7 @@ func (c SystemContracts) All() []SystemContract { // EVMStorage is not included here, since it is not a contract c.Burner, + c.Crypto, } } @@ -366,6 +370,7 @@ func init() { AccountNameEVMStorage: evmStorageEVMFunc, ContractNameBurner: burnerAddressFunc, + ContractNameCrypto: serviceAddressFunc, } getSystemContractsForChain := func(chainID flow.ChainID) *SystemContracts { @@ -421,6 +426,7 @@ func init() { EVMStorage: addressOfAccount(AccountNameEVMStorage), Burner: addressOfContract(ContractNameBurner), + Crypto: addressOfContract(ContractNameCrypto), } return contracts diff --git a/go.mod b/go.mod index 8ac883df3aa..c9beac7c5b5 100644 --- a/go.mod +++ b/go.mod @@ -48,12 +48,12 @@ require ( github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 github.com/onflow/atree v0.8.0 - github.com/onflow/cadence v1.1.0 + github.com/onflow/cadence v1.2.1 github.com/onflow/crypto v0.25.2 github.com/onflow/flow v0.3.4 - github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.2 - github.com/onflow/flow-core-contracts/lib/go/templates v1.3.2 - github.com/onflow/flow-go-sdk v1.1.0 + github.com/onflow/flow-core-contracts/lib/go/contracts v1.4.0 + github.com/onflow/flow-core-contracts/lib/go/templates v1.4.0 + github.com/onflow/flow-go-sdk v1.2.2 github.com/onflow/flow/protobuf/go/flow v0.4.7 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pierrec/lz4 v2.6.1+incompatible diff --git a/go.sum b/go.sum index 79da2652571..036ed5849f6 100644 --- a/go.sum +++ b/go.sum @@ -913,22 +913,22 @@ github.com/onflow/atree v0.8.0 h1:qg5c6J1gVDNObughpEeWm8oxqhPGdEyGrda121GM4u0= github.com/onflow/atree v0.8.0/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483 h1:LpiQhTAfM9CAmNVEs0n//cBBgCg+vJSiIxTHYUklZ84= github.com/onflow/boxo v0.0.0-20240201202436-f2477b92f483/go.mod h1:pIZgTWdm3k3pLF9Uq6MB8JEcW07UDwNJjlXW1HELW80= -github.com/onflow/cadence v1.1.0 h1:wPg86IX1kRv6DWjdETxKa4tv6A3iAwJVzzKAwaGdtDA= -github.com/onflow/cadence v1.1.0/go.mod h1:fJxxOAp1wnWDfOHT8GOc1ypsU0RR5E3z51AhG8Yf5jg= +github.com/onflow/cadence v1.2.1 h1:hmSsgX3rTsp2E5qTSl1JXINt8qepdRrHTwDSYqN5Nxs= +github.com/onflow/cadence v1.2.1/go.mod h1:fJxxOAp1wnWDfOHT8GOc1ypsU0RR5E3z51AhG8Yf5jg= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= github.com/onflow/flow v0.3.4 h1:FXUWVdYB90f/rjNcY0Owo30gL790tiYff9Pb/sycXYE= github.com/onflow/flow v0.3.4/go.mod h1:lzyAYmbu1HfkZ9cfnL5/sjrrsnJiUU8fRL26CqLP7+c= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.2 h1:My0EZLNXlzjN5tT81wD+wNb+PSqfOLJKHA1crXBNu5U= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.2/go.mod h1:9asTBnB6Tw2UlVVtQKyS/egYv3xr4zVlJnJ75z1dfac= -github.com/onflow/flow-core-contracts/lib/go/templates v1.3.2 h1:YNHTonFhPSCuPJFYhIYZ1okSZmXEeQ07eKaoryrjHQI= -github.com/onflow/flow-core-contracts/lib/go/templates v1.3.2/go.mod h1:pN768Al/wLRlf3bwugv9TyxniqJxMu4sxnX9eQJam64= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.4.0 h1:R86HaOuk6vpuECZnriEUE7bw9inC2AtdSn8lL/iwQLQ= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.4.0/go.mod h1:9asTBnB6Tw2UlVVtQKyS/egYv3xr4zVlJnJ75z1dfac= +github.com/onflow/flow-core-contracts/lib/go/templates v1.4.0 h1:u2DAG8pk0xFH7TwS70t1gSZ/FtIIZWMSNyiu4SeXBYg= +github.com/onflow/flow-core-contracts/lib/go/templates v1.4.0/go.mod h1:pN768Al/wLRlf3bwugv9TyxniqJxMu4sxnX9eQJam64= github.com/onflow/flow-ft/lib/go/contracts v1.0.1 h1:Ts5ob+CoCY2EjEd0W6vdLJ7hLL3SsEftzXG2JlmSe24= github.com/onflow/flow-ft/lib/go/contracts v1.0.1/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= github.com/onflow/flow-ft/lib/go/templates v1.0.1 h1:FDYKAiGowABtoMNusLuRCILIZDtVqJ/5tYI4VkF5zfM= github.com/onflow/flow-ft/lib/go/templates v1.0.1/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= -github.com/onflow/flow-go-sdk v1.1.0 h1:DT8P3B3oAicOOXugdev4s1IEKHsiLS9T7MovFcTzB2s= -github.com/onflow/flow-go-sdk v1.1.0/go.mod h1:21g1pqP9Wy8RBXdenNsjzADwbtWNOViUCnfNZwr3trM= +github.com/onflow/flow-go-sdk v1.2.2 h1:F78Sq/VaExgtaQv739k06gnx2aIyLF5wVE0XwxFpmsc= +github.com/onflow/flow-go-sdk v1.2.2/go.mod h1:yhQ5+Sp2xWoCQ1fuRDswawTDQ0ng0z5nTkFVH82xL7E= github.com/onflow/flow-nft/lib/go/contracts v1.2.2 h1:XFERNVUDGbZ4ViZjt7P1cGD80mO1PzUJYPfdhXFsGbQ= github.com/onflow/flow-nft/lib/go/contracts v1.2.2/go.mod h1:eZ9VMMNfCq0ho6kV25xJn1kXeCfxnkhj3MwF3ed08gY= github.com/onflow/flow-nft/lib/go/templates v1.2.1 h1:SAALMZPDw9Eb9p5kSLnmnFxjyig1MLiT4JUlLp0/bSE= diff --git a/insecure/go.mod b/insecure/go.mod index 33d8e855071..f6381343701 100644 --- a/insecure/go.mod +++ b/insecure/go.mod @@ -203,12 +203,12 @@ require ( github.com/nxadm/tail v1.4.8 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/onflow/atree v0.8.0 // indirect - github.com/onflow/cadence v1.1.0 // indirect - github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.2 // indirect - github.com/onflow/flow-core-contracts/lib/go/templates v1.3.2 // indirect + github.com/onflow/cadence v1.2.1 // indirect + github.com/onflow/flow-core-contracts/lib/go/contracts v1.4.0 // indirect + github.com/onflow/flow-core-contracts/lib/go/templates v1.4.0 // indirect github.com/onflow/flow-ft/lib/go/contracts v1.0.1 // indirect github.com/onflow/flow-ft/lib/go/templates v1.0.1 // indirect - github.com/onflow/flow-go-sdk v1.1.0 // indirect + github.com/onflow/flow-go-sdk v1.2.2 // indirect github.com/onflow/flow-nft/lib/go/contracts v1.2.2 // indirect github.com/onflow/flow-nft/lib/go/templates v1.2.1 // indirect github.com/onflow/flow/protobuf/go/flow v0.4.7 // indirect diff --git a/insecure/go.sum b/insecure/go.sum index 27949c5063a..11976c45d95 100644 --- a/insecure/go.sum +++ b/insecure/go.sum @@ -856,20 +856,20 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onflow/atree v0.8.0 h1:qg5c6J1gVDNObughpEeWm8oxqhPGdEyGrda121GM4u0= github.com/onflow/atree v0.8.0/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= -github.com/onflow/cadence v1.1.0 h1:wPg86IX1kRv6DWjdETxKa4tv6A3iAwJVzzKAwaGdtDA= -github.com/onflow/cadence v1.1.0/go.mod h1:fJxxOAp1wnWDfOHT8GOc1ypsU0RR5E3z51AhG8Yf5jg= +github.com/onflow/cadence v1.2.1 h1:hmSsgX3rTsp2E5qTSl1JXINt8qepdRrHTwDSYqN5Nxs= +github.com/onflow/cadence v1.2.1/go.mod h1:fJxxOAp1wnWDfOHT8GOc1ypsU0RR5E3z51AhG8Yf5jg= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.2 h1:My0EZLNXlzjN5tT81wD+wNb+PSqfOLJKHA1crXBNu5U= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.2/go.mod h1:9asTBnB6Tw2UlVVtQKyS/egYv3xr4zVlJnJ75z1dfac= -github.com/onflow/flow-core-contracts/lib/go/templates v1.3.2 h1:YNHTonFhPSCuPJFYhIYZ1okSZmXEeQ07eKaoryrjHQI= -github.com/onflow/flow-core-contracts/lib/go/templates v1.3.2/go.mod h1:pN768Al/wLRlf3bwugv9TyxniqJxMu4sxnX9eQJam64= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.4.0 h1:R86HaOuk6vpuECZnriEUE7bw9inC2AtdSn8lL/iwQLQ= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.4.0/go.mod h1:9asTBnB6Tw2UlVVtQKyS/egYv3xr4zVlJnJ75z1dfac= +github.com/onflow/flow-core-contracts/lib/go/templates v1.4.0 h1:u2DAG8pk0xFH7TwS70t1gSZ/FtIIZWMSNyiu4SeXBYg= +github.com/onflow/flow-core-contracts/lib/go/templates v1.4.0/go.mod h1:pN768Al/wLRlf3bwugv9TyxniqJxMu4sxnX9eQJam64= github.com/onflow/flow-ft/lib/go/contracts v1.0.1 h1:Ts5ob+CoCY2EjEd0W6vdLJ7hLL3SsEftzXG2JlmSe24= github.com/onflow/flow-ft/lib/go/contracts v1.0.1/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= github.com/onflow/flow-ft/lib/go/templates v1.0.1 h1:FDYKAiGowABtoMNusLuRCILIZDtVqJ/5tYI4VkF5zfM= github.com/onflow/flow-ft/lib/go/templates v1.0.1/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= -github.com/onflow/flow-go-sdk v1.1.0 h1:DT8P3B3oAicOOXugdev4s1IEKHsiLS9T7MovFcTzB2s= -github.com/onflow/flow-go-sdk v1.1.0/go.mod h1:21g1pqP9Wy8RBXdenNsjzADwbtWNOViUCnfNZwr3trM= +github.com/onflow/flow-go-sdk v1.2.2 h1:F78Sq/VaExgtaQv739k06gnx2aIyLF5wVE0XwxFpmsc= +github.com/onflow/flow-go-sdk v1.2.2/go.mod h1:yhQ5+Sp2xWoCQ1fuRDswawTDQ0ng0z5nTkFVH82xL7E= github.com/onflow/flow-nft/lib/go/contracts v1.2.2 h1:XFERNVUDGbZ4ViZjt7P1cGD80mO1PzUJYPfdhXFsGbQ= github.com/onflow/flow-nft/lib/go/contracts v1.2.2/go.mod h1:eZ9VMMNfCq0ho6kV25xJn1kXeCfxnkhj3MwF3ed08gY= github.com/onflow/flow-nft/lib/go/templates v1.2.1 h1:SAALMZPDw9Eb9p5kSLnmnFxjyig1MLiT4JUlLp0/bSE= diff --git a/integration/benchmark/account/account_provider.go b/integration/benchmark/account/account_provider.go index cdbd743d025..d46ecd42dbe 100644 --- a/integration/benchmark/account/account_provider.go +++ b/integration/benchmark/account/account_provider.go @@ -10,9 +10,11 @@ import ( "golang.org/x/sync/errgroup" flowsdk "github.com/onflow/flow-go-sdk" + "github.com/onflow/flow-go/module/util" "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go/fvm/blueprints" "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/integration/benchmark/common" diff --git a/integration/benchmark/contLoadGenerator.go b/integration/benchmark/contLoadGenerator.go index 2ca1cee974c..7f1c31562cf 100644 --- a/integration/benchmark/contLoadGenerator.go +++ b/integration/benchmark/contLoadGenerator.go @@ -12,6 +12,7 @@ import ( flowsdk "github.com/onflow/flow-go-sdk" "github.com/onflow/flow-go-sdk/access" "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/integration/benchmark/account" "github.com/onflow/flow-go/integration/benchmark/common" diff --git a/integration/benchmark/follower.go b/integration/benchmark/follower.go index 746c5b17b40..0681570b491 100644 --- a/integration/benchmark/follower.go +++ b/integration/benchmark/follower.go @@ -10,6 +10,7 @@ import ( flowsdk "github.com/onflow/flow-go-sdk" "github.com/onflow/flow-go-sdk/access" + "github.com/onflow/flow-go/module/metrics" "github.com/rs/zerolog" diff --git a/integration/benchmark/follower_test.go b/integration/benchmark/follower_test.go index 1b5b942e497..8ae218a6885 100644 --- a/integration/benchmark/follower_test.go +++ b/integration/benchmark/follower_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" flowsdk "github.com/onflow/flow-go-sdk" + mockClient "github.com/onflow/flow-go/integration/benchmark/mock" "github.com/onflow/flow-go/utils/unittest" ) diff --git a/integration/benchmark/load/common.go b/integration/benchmark/load/common.go index 93882d96757..4ce4e589fc8 100644 --- a/integration/benchmark/load/common.go +++ b/integration/benchmark/load/common.go @@ -7,6 +7,7 @@ import ( "github.com/rs/zerolog" flowsdk "github.com/onflow/flow-go-sdk" + "github.com/onflow/flow-go/integration/benchmark/common" "github.com/onflow/flow-go/integration/benchmark/account" diff --git a/integration/benchmark/load/simple_load.go b/integration/benchmark/load/simple_load.go index e10927d5493..b1b05cf9005 100644 --- a/integration/benchmark/load/simple_load.go +++ b/integration/benchmark/load/simple_load.go @@ -7,6 +7,7 @@ import ( "github.com/rs/zerolog" flowsdk "github.com/onflow/flow-go-sdk" + "github.com/onflow/flow-go/fvm/blueprints" "github.com/onflow/flow-go/integration/benchmark/account" ) diff --git a/integration/benchmark/load/token_transfer_load.go b/integration/benchmark/load/token_transfer_load.go index e382c58611f..703b013900b 100644 --- a/integration/benchmark/load/token_transfer_load.go +++ b/integration/benchmark/load/token_transfer_load.go @@ -5,6 +5,7 @@ import ( "github.com/rs/zerolog" flowsdk "github.com/onflow/flow-go-sdk" + "github.com/onflow/flow-go/fvm/errors" "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/integration/benchmark/account" diff --git a/integration/benchmark/load/token_transfer_multiple_load.go b/integration/benchmark/load/token_transfer_multiple_load.go index 3e7ab6930a1..05bd4d6ca5b 100644 --- a/integration/benchmark/load/token_transfer_multiple_load.go +++ b/integration/benchmark/load/token_transfer_multiple_load.go @@ -9,6 +9,7 @@ import ( "github.com/onflow/flow-go/model/flow" flowsdk "github.com/onflow/flow-go-sdk" + "github.com/onflow/flow-go/fvm/systemcontracts" "github.com/onflow/flow-go/integration/benchmark/account" "github.com/onflow/flow-go/integration/benchmark/scripts" diff --git a/integration/benchmark/scripts/scripts.go b/integration/benchmark/scripts/scripts.go index 105b5285c1f..b21b564ee20 100644 --- a/integration/benchmark/scripts/scripts.go +++ b/integration/benchmark/scripts/scripts.go @@ -8,6 +8,7 @@ import ( "github.com/onflow/cadence" flowsdk "github.com/onflow/flow-go-sdk" + "github.com/onflow/flow-go/model/flow" ) diff --git a/integration/dkg/node.go b/integration/dkg/node.go index cbea2b7f44a..5c16acba499 100644 --- a/integration/dkg/node.go +++ b/integration/dkg/node.go @@ -8,6 +8,7 @@ import ( sdk "github.com/onflow/flow-go-sdk" sdkcrypto "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go/engine/consensus/dkg" testmock "github.com/onflow/flow-go/engine/testutil/mock" "github.com/onflow/flow-go/model/bootstrap" diff --git a/integration/go.mod b/integration/go.mod index 8e5e807dc01..92956116a7e 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -20,13 +20,13 @@ require ( github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ds-pebble v0.3.1-0.20240828032824-d745b9d3200b github.com/libp2p/go-libp2p v0.32.2 - github.com/onflow/cadence v1.1.0 + github.com/onflow/cadence v1.2.1 github.com/onflow/crypto v0.25.2 - github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.2 - github.com/onflow/flow-core-contracts/lib/go/templates v1.3.2 - github.com/onflow/flow-emulator v1.0.2-0.20241018193523-2601797fe0f2 - github.com/onflow/flow-go v0.38.0-preview.0.0.20241018193026-4b778232480b - github.com/onflow/flow-go-sdk v1.1.0 + github.com/onflow/flow-core-contracts/lib/go/contracts v1.4.0 + github.com/onflow/flow-core-contracts/lib/go/templates v1.4.0 + github.com/onflow/flow-emulator v1.0.2-0.20241021223526-a545558d37a2 + github.com/onflow/flow-go v0.38.0-preview.0.0.20241021221952-af9cd6e99de1 + github.com/onflow/flow-go-sdk v1.2.2 github.com/onflow/flow-go/insecure v0.0.0-00010101000000-000000000000 github.com/onflow/flow/protobuf/go/flow v0.4.7 github.com/onflow/go-ethereum v1.14.7 diff --git a/integration/go.sum b/integration/go.sum index 7033bc72121..7e114711f03 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -742,22 +742,22 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onflow/atree v0.8.0 h1:qg5c6J1gVDNObughpEeWm8oxqhPGdEyGrda121GM4u0= github.com/onflow/atree v0.8.0/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= -github.com/onflow/cadence v1.1.0 h1:wPg86IX1kRv6DWjdETxKa4tv6A3iAwJVzzKAwaGdtDA= -github.com/onflow/cadence v1.1.0/go.mod h1:fJxxOAp1wnWDfOHT8GOc1ypsU0RR5E3z51AhG8Yf5jg= +github.com/onflow/cadence v1.2.1 h1:hmSsgX3rTsp2E5qTSl1JXINt8qepdRrHTwDSYqN5Nxs= +github.com/onflow/cadence v1.2.1/go.mod h1:fJxxOAp1wnWDfOHT8GOc1ypsU0RR5E3z51AhG8Yf5jg= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.2 h1:My0EZLNXlzjN5tT81wD+wNb+PSqfOLJKHA1crXBNu5U= -github.com/onflow/flow-core-contracts/lib/go/contracts v1.3.2/go.mod h1:9asTBnB6Tw2UlVVtQKyS/egYv3xr4zVlJnJ75z1dfac= -github.com/onflow/flow-core-contracts/lib/go/templates v1.3.2 h1:YNHTonFhPSCuPJFYhIYZ1okSZmXEeQ07eKaoryrjHQI= -github.com/onflow/flow-core-contracts/lib/go/templates v1.3.2/go.mod h1:pN768Al/wLRlf3bwugv9TyxniqJxMu4sxnX9eQJam64= -github.com/onflow/flow-emulator v1.0.2-0.20241018193523-2601797fe0f2 h1:bCp07I6BKfDGu8CFiWS0OY7FiqELAZNbeZ2M70o1gXY= -github.com/onflow/flow-emulator v1.0.2-0.20241018193523-2601797fe0f2/go.mod h1:QMKsUPYQh4PPvSF3ryYzkbL6HzWAhoRr55FtbKCC5Jc= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.4.0 h1:R86HaOuk6vpuECZnriEUE7bw9inC2AtdSn8lL/iwQLQ= +github.com/onflow/flow-core-contracts/lib/go/contracts v1.4.0/go.mod h1:9asTBnB6Tw2UlVVtQKyS/egYv3xr4zVlJnJ75z1dfac= +github.com/onflow/flow-core-contracts/lib/go/templates v1.4.0 h1:u2DAG8pk0xFH7TwS70t1gSZ/FtIIZWMSNyiu4SeXBYg= +github.com/onflow/flow-core-contracts/lib/go/templates v1.4.0/go.mod h1:pN768Al/wLRlf3bwugv9TyxniqJxMu4sxnX9eQJam64= +github.com/onflow/flow-emulator v1.0.2-0.20241021223526-a545558d37a2 h1:zs3/ctgI1KNFTcA1HpkDmyNuUybY/oFXDE1S+cYR9lU= +github.com/onflow/flow-emulator v1.0.2-0.20241021223526-a545558d37a2/go.mod h1:HgcZT9TZVOqmNGqN3fX+QqoiVovIHfLwqbXwuaEJiXE= github.com/onflow/flow-ft/lib/go/contracts v1.0.1 h1:Ts5ob+CoCY2EjEd0W6vdLJ7hLL3SsEftzXG2JlmSe24= github.com/onflow/flow-ft/lib/go/contracts v1.0.1/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A= github.com/onflow/flow-ft/lib/go/templates v1.0.1 h1:FDYKAiGowABtoMNusLuRCILIZDtVqJ/5tYI4VkF5zfM= github.com/onflow/flow-ft/lib/go/templates v1.0.1/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE= -github.com/onflow/flow-go-sdk v1.1.0 h1:DT8P3B3oAicOOXugdev4s1IEKHsiLS9T7MovFcTzB2s= -github.com/onflow/flow-go-sdk v1.1.0/go.mod h1:21g1pqP9Wy8RBXdenNsjzADwbtWNOViUCnfNZwr3trM= +github.com/onflow/flow-go-sdk v1.2.2 h1:F78Sq/VaExgtaQv739k06gnx2aIyLF5wVE0XwxFpmsc= +github.com/onflow/flow-go-sdk v1.2.2/go.mod h1:yhQ5+Sp2xWoCQ1fuRDswawTDQ0ng0z5nTkFVH82xL7E= github.com/onflow/flow-nft/lib/go/contracts v1.2.2 h1:XFERNVUDGbZ4ViZjt7P1cGD80mO1PzUJYPfdhXFsGbQ= github.com/onflow/flow-nft/lib/go/contracts v1.2.2/go.mod h1:eZ9VMMNfCq0ho6kV25xJn1kXeCfxnkhj3MwF3ed08gY= github.com/onflow/flow-nft/lib/go/templates v1.2.1 h1:SAALMZPDw9Eb9p5kSLnmnFxjyig1MLiT4JUlLp0/bSE= diff --git a/integration/localnet/builder/bootstrap.go b/integration/localnet/builder/bootstrap.go index 200926bf83a..e3ed2c91ea2 100644 --- a/integration/localnet/builder/bootstrap.go +++ b/integration/localnet/builder/bootstrap.go @@ -454,8 +454,8 @@ func prepareObserverService(i int, observerName string, agPublicKey string) Serv service := defaultService(observerName, DefaultObserverRole, dataDir, profilerDir, i) service.Command = append(service.Command, - fmt.Sprintf("--bootstrap-node-addresses=%s:%s", testnet.PrimaryAN, testnet.PublicNetworkPort), - fmt.Sprintf("--bootstrap-node-public-keys=%s", agPublicKey), + fmt.Sprintf("--observer-mode-bootstrap-node-addresses=%s:%s", testnet.PrimaryAN, testnet.PublicNetworkPort), + fmt.Sprintf("--observer-mode-bootstrap-node-public-keys=%s", agPublicKey), fmt.Sprintf("--upstream-node-addresses=%s:%s", testnet.PrimaryAN, testnet.GRPCSecurePort), fmt.Sprintf("--upstream-node-public-keys=%s", agPublicKey), fmt.Sprintf("--observer-networking-key-path=/bootstrap/private-root-information/%s_key", observerName), diff --git a/integration/testnet/network.go b/integration/testnet/network.go index bc5fdc2f48b..250e7eabe53 100644 --- a/integration/testnet/network.go +++ b/integration/testnet/network.go @@ -31,6 +31,7 @@ import ( "github.com/onflow/cadence" "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go/cmd/bootstrap/dkg" "github.com/onflow/flow-go/cmd/bootstrap/run" "github.com/onflow/flow-go/cmd/bootstrap/utils" @@ -756,8 +757,8 @@ func (net *FlowNetwork) AddObserver(t *testing.T, conf ObserverConfig) *Containe fmt.Sprintf("--secretsdir=%s", DefaultFlowSecretsDBDir), fmt.Sprintf("--profiler-dir=%s", DefaultProfilerDir), fmt.Sprintf("--loglevel=%s", conf.LogLevel.String()), - fmt.Sprintf("--bootstrap-node-addresses=%s", accessNode.ContainerAddr(PublicNetworkPort)), - fmt.Sprintf("--bootstrap-node-public-keys=%s", accessPublicKey), + fmt.Sprintf("--observer-mode-bootstrap-node-addresses=%s", accessNode.ContainerAddr(PublicNetworkPort)), + fmt.Sprintf("--observer-mode-bootstrap-node-public-keys=%s", accessPublicKey), fmt.Sprintf("--upstream-node-addresses=%s", accessNode.ContainerAddr(GRPCSecurePort)), fmt.Sprintf("--upstream-node-public-keys=%s", accessPublicKey), fmt.Sprintf("--observer-networking-key-path=%s/private-root-information/%s_key", DefaultBootstrapDir, conf.ContainerName), diff --git a/integration/tests/access/cohort2/observer_indexer_enabled_test.go b/integration/tests/access/cohort2/observer_indexer_enabled_test.go index 43f784669bc..cc2709f9780 100644 --- a/integration/tests/access/cohort2/observer_indexer_enabled_test.go +++ b/integration/tests/access/cohort2/observer_indexer_enabled_test.go @@ -17,6 +17,7 @@ import ( sdk "github.com/onflow/flow-go-sdk" sdkcrypto "github.com/onflow/flow-go-sdk/crypto" "github.com/onflow/flow-go-sdk/templates" + "github.com/onflow/flow-go/engine/access/rpc/backend" "github.com/onflow/flow-go/engine/common/rpc/convert" "github.com/onflow/flow-go/integration/testnet" diff --git a/integration/tests/access/cohort3/access_circuit_breaker_test.go b/integration/tests/access/cohort3/access_circuit_breaker_test.go index 772a56e6c8c..8cb745bd569 100644 --- a/integration/tests/access/cohort3/access_circuit_breaker_test.go +++ b/integration/tests/access/cohort3/access_circuit_breaker_test.go @@ -16,6 +16,7 @@ import ( sdk "github.com/onflow/flow-go-sdk" sdkcrypto "github.com/onflow/flow-go-sdk/crypto" "github.com/onflow/flow-go-sdk/templates" + "github.com/onflow/flow-go/integration/testnet" "github.com/onflow/flow-go/integration/tests/lib" "github.com/onflow/flow-go/model/flow" diff --git a/integration/tests/access/cohort3/access_store_tx_error_messages_test.go b/integration/tests/access/cohort3/access_store_tx_error_messages_test.go new file mode 100644 index 00000000000..23a0b37f551 --- /dev/null +++ b/integration/tests/access/cohort3/access_store_tx_error_messages_test.go @@ -0,0 +1,199 @@ +package cohort3 + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + sdk "github.com/onflow/flow-go-sdk" + sdkcrypto "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go-sdk/templates" + + "github.com/onflow/flow-go/integration/convert" + "github.com/onflow/flow-go/integration/testnet" + "github.com/onflow/flow-go/integration/tests/lib" + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage/badger" +) + +const maxReceiptHeightMetric = "access_ingestion_max_receipt_height" + +func TestAccessStoreTxErrorMessages(t *testing.T) { + suite.Run(t, new(AccessStoreTxErrorMessagesSuite)) +} + +// AccessStoreTxErrorMessagesSuite tests the access for storing transaction error messages. +type AccessStoreTxErrorMessagesSuite struct { + suite.Suite + + log zerolog.Logger + + // root context for the current test + ctx context.Context + cancel context.CancelFunc + + net *testnet.FlowNetwork +} + +func (s *AccessStoreTxErrorMessagesSuite) TearDownTest() { + s.log.Info().Msg("================> Start TearDownTest") + s.net.Remove() + s.cancel() + s.log.Info().Msg("================> Finish TearDownTest") +} + +// SetupTest sets up the test suite by starting the network. +// The access are started with correct parameters and store transaction error messages. +func (s *AccessStoreTxErrorMessagesSuite) SetupTest() { + defaultAccess := testnet.NewNodeConfig( + flow.RoleAccess, + testnet.WithLogLevel(zerolog.FatalLevel), + ) + + storeTxAccess := testnet.NewNodeConfig( + flow.RoleAccess, + testnet.WithLogLevel(zerolog.InfoLevel), + testnet.WithAdditionalFlagf("--store-tx-result-error-messages=true"), + testnet.WithMetricsServer(), + ) + + consensusConfigs := []func(config *testnet.NodeConfig){ + // `cruise-ctl-fallback-proposal-duration` is set to 250ms instead to of 100ms + // to purposely slow down the block rate. This is needed since the crypto module + // update providing faster BLS operations. + testnet.WithAdditionalFlag("--cruise-ctl-fallback-proposal-duration=250ms"), + testnet.WithAdditionalFlagf("--required-verification-seal-approvals=%d", 1), + testnet.WithAdditionalFlagf("--required-construction-seal-approvals=%d", 1), + testnet.WithLogLevel(zerolog.FatalLevel), + } + + nodeConfigs := []testnet.NodeConfig{ + testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel)), + testnet.NewNodeConfig(flow.RoleCollection, testnet.WithLogLevel(zerolog.FatalLevel)), + testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.FatalLevel)), + testnet.NewNodeConfig(flow.RoleExecution, testnet.WithLogLevel(zerolog.FatalLevel)), + testnet.NewNodeConfig(flow.RoleConsensus, consensusConfigs...), + testnet.NewNodeConfig(flow.RoleConsensus, consensusConfigs...), + testnet.NewNodeConfig(flow.RoleConsensus, consensusConfigs...), + testnet.NewNodeConfig(flow.RoleVerification, testnet.WithLogLevel(zerolog.FatalLevel)), + + defaultAccess, + storeTxAccess, + } + + // prepare the network + conf := testnet.NewNetworkConfig("access_store_tx_error_messages_test", nodeConfigs) + s.net = testnet.PrepareFlowNetwork(s.T(), conf, flow.Localnet) + + // start the network + s.ctx, s.cancel = context.WithCancel(context.Background()) + + s.net.Start(s.ctx) +} + +// TestAccessStoreTxErrorMessages verifies that transaction result error messages +// are stored correctly in the database by sending a transaction, generating an error, +// and checking if the error message is properly stored and retrieved from the database. +func (s *AccessStoreTxErrorMessagesSuite) TestAccessStoreTxErrorMessages() { + // Create and send a transaction that will result in an error. + txResult := s.createAndSendTxWithTxError() + + txBlockID := convert.IDFromSDK(txResult.BlockID) + txID := convert.IDFromSDK(txResult.TransactionID) + expectedTxResultErrorMessage := txResult.Error.Error() + + accessContainerName := "access_2" + + // Wait until execution receipts are handled, transaction error messages are stored. + s.Eventually(func() bool { + value, err := s.getMaxReceiptHeight(accessContainerName) + return err == nil && value > txResult.BlockHeight + }, 60*time.Second, 1*time.Second) + + // Stop the network containers before checking the results. + s.net.StopContainers() + + // Get the access node and open the protocol DB. + accessNode := s.net.ContainerByName(accessContainerName) + // setup storage objects needed to get the execution data id + anDB, err := accessNode.DB() + require.NoError(s.T(), err, "could not open db") + + metrics := metrics.NewNoopCollector() + anTxErrorMessages := badger.NewTransactionResultErrorMessages(metrics, anDB, badger.DefaultCacheSize) + + // Fetch the stored error message by block ID and transaction ID. + errMsgResult, err := anTxErrorMessages.ByBlockIDTransactionID(txBlockID, txID) + s.Require().NoError(err) + + // Verify that the error message retrieved matches the expected values. + s.Require().Equal(txID, errMsgResult.TransactionID) + s.Require().Equal(expectedTxResultErrorMessage, errMsgResult.ErrorMessage) +} + +// createAndSendTxWithTxError creates and sends a transaction that will result in an error. +// This function creates a new account, causing an error during execution. +func (s *AccessStoreTxErrorMessagesSuite) createAndSendTxWithTxError() *sdk.TransactionResult { + // prepare environment to create a new account + serviceAccountClient, err := s.net.ContainerByName(testnet.PrimaryAN).TestnetClient() + s.Require().NoError(err) + + latestBlockID, err := serviceAccountClient.GetLatestBlockID(s.ctx) + s.Require().NoError(err) + + // create new account to deploy Counter to + accountPrivateKey := lib.RandomPrivateKey() + + accountKey := sdk.NewAccountKey(). + FromPrivateKey(accountPrivateKey). + SetHashAlgo(sdkcrypto.SHA3_256). + SetWeight(sdk.AccountKeyWeightThreshold) + + serviceAddress := sdk.Address(serviceAccountClient.Chain.ServiceAddress()) + + // Generate the account creation transaction + createAccountTx, err := templates.CreateAccount( + []*sdk.AccountKey{accountKey}, + []templates.Contract{}, serviceAddress) + s.Require().NoError(err) + + // Generate the account creation transaction + createAccountTx. + SetReferenceBlockID(sdk.Identifier(latestBlockID)). + SetProposalKey(serviceAddress, 1, serviceAccountClient.GetAndIncrementSeqNumber()). + SetPayer(serviceAddress). + SetComputeLimit(9999) + + // Sign and send the transaction. + err = serviceAccountClient.SignAndSendTransaction(s.ctx, createAccountTx) + s.Require().NoError(err) + + // Wait for the transaction to be sealed and return the transaction result. + accountCreationTxRes, err := serviceAccountClient.WaitForSealed(s.ctx, createAccountTx.ID()) + s.Require().NoError(err) + + return accountCreationTxRes +} + +// getMaxReceiptHeight retrieves the maximum receipt height for a given container by +// querying the metrics endpoint. This is used to confirm that the transaction receipts +// have been processed. +func (s *AccessStoreTxErrorMessagesSuite) getMaxReceiptHeight(containerName string) (uint64, error) { + node := s.net.ContainerByName(containerName) + metricsURL := fmt.Sprintf("http://0.0.0.0:%s/metrics", node.Port(testnet.MetricsPort)) + values := s.net.GetMetricFromContainer(s.T(), containerName, metricsURL, maxReceiptHeightMetric) + + // If no values are found in the metrics, return an error. + if len(values) == 0 { + return 0, fmt.Errorf("no values found") + } + + // Return the first value found as the max receipt height. + return uint64(values[0].GetGauge().GetValue()), nil +} diff --git a/integration/tests/collection/recovery_test.go b/integration/tests/collection/recovery_test.go index 6d1309df18c..d1812e83c7a 100644 --- a/integration/tests/collection/recovery_test.go +++ b/integration/tests/collection/recovery_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/suite" client "github.com/onflow/flow-go-sdk/access/grpc" + "github.com/onflow/flow-go/integration/convert" "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/utils/unittest" diff --git a/integration/utils/emulator_client.go b/integration/utils/emulator_client.go index 8d42e1388fd..1af763e6cff 100644 --- a/integration/utils/emulator_client.go +++ b/integration/utils/emulator_client.go @@ -12,6 +12,7 @@ import ( sdk "github.com/onflow/flow-go-sdk" "github.com/onflow/flow-go-sdk/templates" + "github.com/onflow/flow-go/model/flow" ) diff --git a/ledger/common/bitutils/utils_test.go b/ledger/common/bitutils/utils_test.go index d8f23dfd1a4..f168c058ffa 100644 --- a/ledger/common/bitutils/utils_test.go +++ b/ledger/common/bitutils/utils_test.go @@ -5,9 +5,8 @@ import ( "math/big" "math/bits" "math/rand" - "time" - "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/model/flow/transaction_result.go b/model/flow/transaction_result.go index 3a327118165..f4506c47039 100644 --- a/model/flow/transaction_result.go +++ b/model/flow/transaction_result.go @@ -52,3 +52,17 @@ type LightTransactionResult struct { // ComputationUsed is amount of computation used while executing the transaction. ComputationUsed uint64 } + +// TransactionResultErrorMessage represents an error message resulting from a transaction's execution. +// This struct holds the transaction's ID, its index, any error message generated during execution, +// and the identifier of the execution node that provided the error message. +type TransactionResultErrorMessage struct { + // TransactionID is the ID of the transaction this result error was emitted from. + TransactionID Identifier + // Index is the index of the transaction this result error was emitted from. + Index uint32 + // ErrorMessage contains the error message of any error that may have occurred when the transaction was executed. + ErrorMessage string + // Executor node ID of the execution node that the message was received from. + ExecutorID Identifier +} diff --git a/model/flow/transaction_timing.go b/model/flow/transaction_timing.go index 5f2c58812de..3a9da43eee1 100644 --- a/model/flow/transaction_timing.go +++ b/model/flow/transaction_timing.go @@ -10,6 +10,7 @@ type TransactionTiming struct { Received time.Time Finalized time.Time Executed time.Time + Sealed time.Time } func (t TransactionTiming) ID() Identifier { diff --git a/module/epochs/base_client.go b/module/epochs/base_client.go index 5b372d80141..a0b845fd19a 100644 --- a/module/epochs/base_client.go +++ b/module/epochs/base_client.go @@ -12,6 +12,7 @@ import ( sdk "github.com/onflow/flow-go-sdk" sdkcrypto "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/module" diff --git a/module/epochs/qc_client.go b/module/epochs/qc_client.go index 8bdf32d6a57..4f891361063 100644 --- a/module/epochs/qc_client.go +++ b/module/epochs/qc_client.go @@ -13,6 +13,7 @@ import ( sdk "github.com/onflow/flow-go-sdk" sdkcrypto "github.com/onflow/flow-go-sdk/crypto" + "github.com/onflow/flow-go/network" "github.com/onflow/flow-go/consensus/hotstuff/model" diff --git a/module/jobqueue.go b/module/jobqueue.go index a87208a94f4..9150956504b 100644 --- a/module/jobqueue.go +++ b/module/jobqueue.go @@ -13,8 +13,9 @@ const ( ConsumeProgressExecutionDataIndexerBlockHeight = "ConsumeProgressExecutionDataIndexerBlockHeight" - ConsumeProgressIngestionEngineBlockHeight = "ConsumeProgressIngestionEngineBlockHeight" - ConsumeProgressLastFullBlockHeight = "ConsumeProgressLastFullBlockHeight" + ConsumeProgressIngestionEngineBlockHeight = "ConsumeProgressIngestionEngineBlockHeight" + ConsumeProgressEngineTxErrorMessagesBlockHeight = "ConsumeProgressEngineTxErrorMessagesBlockHeight" + ConsumeProgressLastFullBlockHeight = "ConsumeProgressLastFullBlockHeight" ) // JobID is a unique ID of the job. diff --git a/module/metrics.go b/module/metrics.go index f43c8b9325e..9727da98afc 100644 --- a/module/metrics.go +++ b/module/metrics.go @@ -1076,6 +1076,10 @@ type TransactionMetrics interface { // works if the transaction was earlier added as received. TransactionFinalized(txID flow.Identifier, when time.Time) + // TransactionSealed reports the time spent between the transaction being received and sealed. Reporting only + // works if the transaction was earlier added as received. + TransactionSealed(txID flow.Identifier, when time.Time) + // TransactionExecuted reports the time spent between the transaction being received and executed. Reporting only // works if the transaction was earlier added as received. TransactionExecuted(txID flow.Identifier, when time.Time) diff --git a/module/metrics/labels.go b/module/metrics/labels.go index 82260ca3c5d..20b66ad7d68 100644 --- a/module/metrics/labels.go +++ b/module/metrics/labels.go @@ -109,28 +109,30 @@ const ( ResourceNetworkingUnicastDialConfigCache = "unicast_dial_config_cache" ResourceNetworkingGossipsubDuplicateMessagesTrackerCache = "gossipsub_duplicate_messages_tracker_cache" - ResourceFollowerPendingBlocksCache = "follower_pending_block_cache" // follower engine - ResourceFollowerLoopCertifiedBlocksChannel = "follower_loop_certified_blocks_channel" // follower loop, certified blocks buffered channel - ResourceClusterBlockProposalQueue = "cluster_compliance_proposal_queue" // collection node, compliance engine - ResourceTransactionIngestQueue = "ingest_transaction_queue" // collection node, ingest engine - ResourceBeaconKey = "beacon-key" // consensus node, DKG engine - ResourceDKGMessage = "dkg_private_message" // consensus, DKG messaging engine - ResourceApprovalQueue = "sealing_approval_queue" // consensus node, sealing engine - ResourceReceiptQueue = "sealing_receipt_queue" // consensus node, sealing engine - ResourceApprovalResponseQueue = "sealing_approval_response_queue" // consensus node, sealing engine - ResourceBlockResponseQueue = "compliance_block_response_queue" // consensus node, compliance engine - ResourceBlockProposalQueue = "compliance_proposal_queue" // consensus node, compliance engine - ResourceBlockVoteQueue = "vote_aggregator_queue" // consensus/collection node, vote aggregator - ResourceTimeoutObjectQueue = "timeout_aggregator_queue" // consensus/collection node, timeout aggregator - ResourceCollectionGuaranteesQueue = "ingestion_col_guarantee_queue" // consensus node, ingestion engine - ResourceChunkDataPack = "chunk_data_pack" // execution node - ResourceChunkDataPackRequests = "chunk_data_pack_request" // execution node - ResourceEvents = "events" // execution node - ResourceServiceEvents = "service_events" // execution node - ResourceTransactionResults = "transaction_results" // execution node - ResourceTransactionResultIndices = "transaction_result_indices" // execution node - ResourceTransactionResultByBlock = "transaction_result_by_block" // execution node - ResourceExecutionDataCache = "execution_data_cache" // access node + ResourceFollowerPendingBlocksCache = "follower_pending_block_cache" // follower engine + ResourceFollowerLoopCertifiedBlocksChannel = "follower_loop_certified_blocks_channel" // follower loop, certified blocks buffered channel + ResourceClusterBlockProposalQueue = "cluster_compliance_proposal_queue" // collection node, compliance engine + ResourceTransactionIngestQueue = "ingest_transaction_queue" // collection node, ingest engine + ResourceBeaconKey = "beacon-key" // consensus node, DKG engine + ResourceDKGMessage = "dkg_private_message" // consensus, DKG messaging engine + ResourceApprovalQueue = "sealing_approval_queue" // consensus node, sealing engine + ResourceReceiptQueue = "sealing_receipt_queue" // consensus node, sealing engine + ResourceApprovalResponseQueue = "sealing_approval_response_queue" // consensus node, sealing engine + ResourceBlockResponseQueue = "compliance_block_response_queue" // consensus node, compliance engine + ResourceBlockProposalQueue = "compliance_proposal_queue" // consensus node, compliance engine + ResourceBlockVoteQueue = "vote_aggregator_queue" // consensus/collection node, vote aggregator + ResourceTimeoutObjectQueue = "timeout_aggregator_queue" // consensus/collection node, timeout aggregator + ResourceCollectionGuaranteesQueue = "ingestion_col_guarantee_queue" // consensus node, ingestion engine + ResourceChunkDataPack = "chunk_data_pack" // execution node + ResourceChunkDataPackRequests = "chunk_data_pack_request" // execution node + ResourceEvents = "events" // execution node + ResourceServiceEvents = "service_events" // execution node + ResourceTransactionResults = "transaction_results" // execution node + ResourceTransactionResultIndices = "transaction_result_indices" // execution node + ResourceTransactionResultErrorMessages = "transaction_result_error_messages" // execution node + ResourceTransactionResultErrorMessagesIndices = "transaction_result_error_messages_indices" // execution node + ResourceTransactionResultByBlock = "transaction_result_by_block" // execution node + ResourceExecutionDataCache = "execution_data_cache" // access node ) const ( diff --git a/module/metrics/noop.go b/module/metrics/noop.go index 17460bf460a..60d7e17a287 100644 --- a/module/metrics/noop.go +++ b/module/metrics/noop.go @@ -214,6 +214,7 @@ func (nc *NoopCollector) ScriptExecutionNotIndexed() func (nc *NoopCollector) TransactionResultFetched(dur time.Duration, size int) {} func (nc *NoopCollector) TransactionReceived(txID flow.Identifier, when time.Time) {} func (nc *NoopCollector) TransactionFinalized(txID flow.Identifier, when time.Time) {} +func (nc *NoopCollector) TransactionSealed(txID flow.Identifier, when time.Time) {} func (nc *NoopCollector) TransactionExecuted(txID flow.Identifier, when time.Time) {} func (nc *NoopCollector) TransactionExpired(txID flow.Identifier) {} func (nc *NoopCollector) TransactionValidated() {} diff --git a/module/metrics/transaction.go b/module/metrics/transaction.go index 8bea3e9adea..474d1ba5fe1 100644 --- a/module/metrics/transaction.go +++ b/module/metrics/transaction.go @@ -18,9 +18,11 @@ type TransactionCollector struct { logTimeToFinalized bool logTimeToExecuted bool logTimeToFinalizedExecuted bool + logTimeToSealed bool timeToFinalized prometheus.Summary timeToExecuted prometheus.Summary timeToFinalizedExecuted prometheus.Summary + timeToSealed prometheus.Summary transactionSubmission *prometheus.CounterVec transactionSize prometheus.Histogram scriptExecutedDuration *prometheus.HistogramVec @@ -40,6 +42,7 @@ func NewTransactionCollector( logTimeToFinalized bool, logTimeToExecuted bool, logTimeToFinalizedExecuted bool, + logTimeToSealed bool, ) *TransactionCollector { tc := &TransactionCollector{ @@ -48,6 +51,7 @@ func NewTransactionCollector( logTimeToFinalized: logTimeToFinalized, logTimeToExecuted: logTimeToExecuted, logTimeToFinalizedExecuted: logTimeToFinalizedExecuted, + logTimeToSealed: logTimeToSealed, timeToFinalized: promauto.NewSummary(prometheus.SummaryOpts{ Name: "time_to_finalized_seconds", Namespace: namespaceAccess, @@ -91,6 +95,20 @@ func NewTransactionCollector( AgeBuckets: 5, BufCap: 500, }), + timeToSealed: promauto.NewSummary(prometheus.SummaryOpts{ + Name: "time_to_seal_seconds", + Namespace: namespaceAccess, + Subsystem: subsystemTransactionTiming, + Help: "the duration of how long it took between the transaction was received until it was sealed", + Objectives: map[float64]float64{ + 0.01: 0.001, + 0.5: 0.05, + 0.99: 0.001, + }, + MaxAge: 10 * time.Minute, + AgeBuckets: 5, + BufCap: 500, + }), transactionSubmission: promauto.NewCounterVec(prometheus.CounterOpts{ Name: "transaction_submission", Namespace: namespaceAccess, @@ -269,6 +287,27 @@ func (tc *TransactionCollector) TransactionExecuted(txID flow.Identifier, when t } } +func (tc *TransactionCollector) TransactionSealed(txID flow.Identifier, when time.Time) { + t, updated := tc.transactionTimings.Adjust(txID, func(t *flow.TransactionTiming) *flow.TransactionTiming { + t.Sealed = when + return t + }) + + if !updated { + tc.log.Debug(). + Str("transaction_id", txID.String()). + Msg("failed to update TransactionSealed metric") + return + } + + tc.trackTTS(t, tc.logTimeToSealed) + + // remove transaction timing from mempool if sealed + if !t.Sealed.IsZero() { + tc.transactionTimings.Remove(txID) + } +} + func (tc *TransactionCollector) trackTTF(t *flow.TransactionTiming, log bool) { if t.Received.IsZero() || t.Finalized.IsZero() { return @@ -317,6 +356,20 @@ func (tc *TransactionCollector) trackTTFE(t *flow.TransactionTiming, log bool) { } } +func (tc *TransactionCollector) trackTTS(t *flow.TransactionTiming, log bool) { + if t.Received.IsZero() || t.Sealed.IsZero() { + return + } + duration := t.Sealed.Sub(t.Received).Seconds() + + tc.timeToSealed.Observe(duration) + + if log { + tc.log.Info().Str("transaction_id", t.TransactionID.String()).Float64("duration", duration). + Msg("transaction time to sealed") + } +} + func (tc *TransactionCollector) TransactionSubmissionFailed() { tc.transactionSubmission.WithLabelValues("failed").Inc() } diff --git a/module/mock/access_metrics.go b/module/mock/access_metrics.go index 21ecc03740f..df3cb8ad8c2 100644 --- a/module/mock/access_metrics.go +++ b/module/mock/access_metrics.go @@ -138,6 +138,11 @@ func (_m *AccessMetrics) TransactionResultFetched(dur time.Duration, size int) { _m.Called(dur, size) } +// TransactionSealed provides a mock function with given fields: txID, when +func (_m *AccessMetrics) TransactionSealed(txID flow.Identifier, when time.Time) { + _m.Called(txID, when) +} + // TransactionSubmissionFailed provides a mock function with given fields: func (_m *AccessMetrics) TransactionSubmissionFailed() { _m.Called() diff --git a/module/mock/transaction_metrics.go b/module/mock/transaction_metrics.go index 9345b934a9a..5e96e52a4ad 100644 --- a/module/mock/transaction_metrics.go +++ b/module/mock/transaction_metrics.go @@ -39,6 +39,11 @@ func (_m *TransactionMetrics) TransactionResultFetched(dur time.Duration, size i _m.Called(dur, size) } +// TransactionSealed provides a mock function with given fields: txID, when +func (_m *TransactionMetrics) TransactionSealed(txID flow.Identifier, when time.Time) { + _m.Called(txID, when) +} + // TransactionSubmissionFailed provides a mock function with given fields: func (_m *TransactionMetrics) TransactionSubmissionFailed() { _m.Called() diff --git a/module/state_synchronization/indexer/collection_executed_metric.go b/module/state_synchronization/indexer/collection_executed_metric.go index 814afbb3325..bc1ee3fd341 100644 --- a/module/state_synchronization/indexer/collection_executed_metric.go +++ b/module/state_synchronization/indexer/collection_executed_metric.go @@ -25,6 +25,8 @@ type CollectionExecutedMetricImpl struct { collections storage.Collections blocks storage.Blocks + + blockTransactions *stdmap.IdentifierMap // Map to track transactions for each block for sealed metrics } func NewCollectionExecutedMetricImpl( @@ -35,6 +37,7 @@ func NewCollectionExecutedMetricImpl( blocksToMarkExecuted *stdmap.Times, collections storage.Collections, blocks storage.Blocks, + blockTransactions *stdmap.IdentifierMap, ) (*CollectionExecutedMetricImpl, error) { return &CollectionExecutedMetricImpl{ log: log, @@ -44,16 +47,32 @@ func NewCollectionExecutedMetricImpl( blocksToMarkExecuted: blocksToMarkExecuted, collections: collections, blocks: blocks, + blockTransactions: blockTransactions, }, nil } // CollectionFinalized tracks collections to mark finalized func (c *CollectionExecutedMetricImpl) CollectionFinalized(light flow.LightCollection) { - if ti, found := c.collectionsToMarkFinalized.ByID(light.ID()); found { + lightID := light.ID() + if ti, found := c.collectionsToMarkFinalized.ByID(lightID); found { + + block, err := c.blocks.ByCollectionID(lightID) + if err != nil { + c.log.Warn().Err(err).Msg("could not find block by collection ID") + return + } + blockID := block.ID() + for _, t := range light.Transactions { c.accessMetrics.TransactionFinalized(t, ti) + + err = c.blockTransactions.Append(blockID, t) + if err != nil { + c.log.Warn().Err(err).Msg("could not append finalized tx to track sealed transactions") + continue + } } - c.collectionsToMarkFinalized.Remove(light.ID()) + c.collectionsToMarkFinalized.Remove(lightID) } } @@ -88,6 +107,24 @@ func (c *CollectionExecutedMetricImpl) BlockFinalized(block *flow.Block) { for _, t := range l.Transactions { c.accessMetrics.TransactionFinalized(t, now) + err = c.blockTransactions.Append(blockID, t) + + if err != nil { + c.log.Warn().Err(err).Msg("could not append finalized tx to track sealed transactions") + continue + } + } + } + + // Process block seals + for _, s := range block.Payload.Seals { + transactions, found := c.blockTransactions.Get(s.BlockID) + + if found { + for _, t := range transactions { + c.accessMetrics.TransactionSealed(t, now) + } + c.blockTransactions.Remove(s.BlockID) } } diff --git a/module/state_synchronization/indexer/indexer_core.go b/module/state_synchronization/indexer/indexer_core.go index 95ddc4fb3b0..22a6d16ea2a 100644 --- a/module/state_synchronization/indexer/indexer_core.go +++ b/module/state_synchronization/indexer/indexer_core.go @@ -326,7 +326,7 @@ func (c *IndexerCore) indexRegisters(registers map[ledger.Path]*ledger.Payload, return c.registers.Store(regEntries, height) } -// HandleCollection handles the response of the a collection request made earlier when a block was received. +// HandleCollection handles the response of the collection request made earlier when a block was received. // No errors expected during normal operations. func HandleCollection( collection *flow.Collection, diff --git a/module/state_synchronization/indexer/indexer_core_test.go b/module/state_synchronization/indexer/indexer_core_test.go index 5fd93d4b824..f446c0740a0 100644 --- a/module/state_synchronization/indexer/indexer_core_test.go +++ b/module/state_synchronization/indexer/indexer_core_test.go @@ -197,6 +197,8 @@ func (i *indexCoreTest) initIndexer() *indexCoreTest { require.NoError(i.t, err) blocksToMarkExecuted, err := stdmap.NewTimes(100) require.NoError(i.t, err) + blockTransactions, err := stdmap.NewIdentifierMap(100) + require.NoError(i.t, err) log := zerolog.New(os.Stdout) blocks := storagemock.NewBlocks(i.t) @@ -209,6 +211,7 @@ func (i *indexCoreTest) initIndexer() *indexCoreTest { blocksToMarkExecuted, i.collections, blocks, + blockTransactions, ) require.NoError(i.t, err) diff --git a/storage/all.go b/storage/all.go index 2d0075aa8be..26bb89bd454 100644 --- a/storage/all.go +++ b/storage/all.go @@ -2,26 +2,27 @@ package storage // All includes all the storage modules type All struct { - Headers Headers - Guarantees Guarantees - Seals Seals - Index Index - Payloads Payloads - Blocks Blocks - QuorumCertificates QuorumCertificates - Setups EpochSetups - EpochCommits EpochCommits - Results ExecutionResults - Receipts ExecutionReceipts - ChunkDataPacks ChunkDataPacks - Commits Commits - Transactions Transactions - LightTransactionResults LightTransactionResults - TransactionResults TransactionResults - Collections Collections - Events Events - EpochProtocolStateEntries EpochProtocolStateEntries - ProtocolKVStore ProtocolKVStore - VersionBeacons VersionBeacons - RegisterIndex RegisterIndex + Headers Headers + Guarantees Guarantees + Seals Seals + Index Index + Payloads Payloads + Blocks Blocks + QuorumCertificates QuorumCertificates + Setups EpochSetups + EpochCommits EpochCommits + Results ExecutionResults + Receipts ExecutionReceipts + ChunkDataPacks ChunkDataPacks + Commits Commits + Transactions Transactions + LightTransactionResults LightTransactionResults + TransactionResults TransactionResults + TransactionResultErrorMessages TransactionResultErrorMessages + Collections Collections + Events Events + EpochProtocolStateEntries EpochProtocolStateEntries + ProtocolKVStore ProtocolKVStore + VersionBeacons VersionBeacons + RegisterIndex RegisterIndex } diff --git a/storage/badger/operation/prefix.go b/storage/badger/operation/prefix.go index ad909faf394..4113e1fcd3f 100644 --- a/storage/badger/operation/prefix.go +++ b/storage/badger/operation/prefix.go @@ -89,20 +89,22 @@ const ( codeJobQueuePointer = 72 // legacy codes (should be cleaned up) - codeChunkDataPack = 100 - codeCommit = 101 - codeEvent = 102 - codeExecutionStateInteractions = 103 - codeTransactionResult = 104 - codeFinalizedCluster = 105 - codeServiceEvent = 106 - codeTransactionResultIndex = 107 - codeLightTransactionResult = 108 - codeLightTransactionResultIndex = 109 - codeIndexCollection = 200 - codeIndexExecutionResultByBlock = 202 - codeIndexCollectionByTransaction = 203 - codeIndexResultApprovalByChunk = 204 + codeChunkDataPack = 100 + codeCommit = 101 + codeEvent = 102 + codeExecutionStateInteractions = 103 + codeTransactionResult = 104 + codeFinalizedCluster = 105 + codeServiceEvent = 106 + codeTransactionResultIndex = 107 + codeLightTransactionResult = 108 + codeLightTransactionResultIndex = 109 + codeTransactionResultErrorMessage = 110 + codeTransactionResultErrorMessageIndex = 111 + codeIndexCollection = 200 + codeIndexExecutionResultByBlock = 202 + codeIndexCollectionByTransaction = 203 + codeIndexResultApprovalByChunk = 204 // TEMPORARY codes blockedNodeIDs = 205 // manual override for adding node IDs to list of ejected nodes, applies to networking layer only diff --git a/storage/badger/operation/transaction_results.go b/storage/badger/operation/transaction_results.go index 7d5fcf47086..c4264640364 100644 --- a/storage/badger/operation/transaction_results.go +++ b/storage/badger/operation/transaction_results.go @@ -120,3 +120,51 @@ func LookupLightTransactionResultsByBlockIDUsingIndex(blockID flow.Identifier, t return traverse(makePrefix(codeLightTransactionResultIndex, blockID), txErrIterFunc) } + +// BatchInsertTransactionResultErrorMessage inserts a transaction result error message by block ID and transaction ID +// into the database using a batch write. +func BatchInsertTransactionResultErrorMessage(blockID flow.Identifier, transactionResultErrorMessage *flow.TransactionResultErrorMessage) func(batch *badger.WriteBatch) error { + return batchWrite(makePrefix(codeTransactionResultErrorMessage, blockID, transactionResultErrorMessage.TransactionID), transactionResultErrorMessage) +} + +// BatchIndexTransactionResultErrorMessage indexes a transaction result error message by index within the block using a +// batch write. +func BatchIndexTransactionResultErrorMessage(blockID flow.Identifier, transactionResultErrorMessage *flow.TransactionResultErrorMessage) func(batch *badger.WriteBatch) error { + return batchWrite(makePrefix(codeTransactionResultErrorMessageIndex, blockID, transactionResultErrorMessage.Index), transactionResultErrorMessage) +} + +// RetrieveTransactionResultErrorMessage retrieves a transaction result error message by block ID and transaction ID. +func RetrieveTransactionResultErrorMessage(blockID flow.Identifier, transactionID flow.Identifier, transactionResultErrorMessage *flow.TransactionResultErrorMessage) func(*badger.Txn) error { + return retrieve(makePrefix(codeTransactionResultErrorMessage, blockID, transactionID), transactionResultErrorMessage) +} + +// RetrieveTransactionResultErrorMessageByIndex retrieves a transaction result error message by block ID and index. +func RetrieveTransactionResultErrorMessageByIndex(blockID flow.Identifier, txIndex uint32, transactionResultErrorMessage *flow.TransactionResultErrorMessage) func(*badger.Txn) error { + return retrieve(makePrefix(codeTransactionResultErrorMessageIndex, blockID, txIndex), transactionResultErrorMessage) +} + +// TransactionResultErrorMessagesExists checks whether tx result error messages exist in the database. +func TransactionResultErrorMessagesExists(blockID flow.Identifier, blockExists *bool) func(*badger.Txn) error { + return exists(makePrefix(codeTransactionResultErrorMessageIndex, blockID), blockExists) +} + +// LookupTransactionResultErrorMessagesByBlockIDUsingIndex retrieves all tx result error messages for a block, by using +// tx_index index. This correctly handles cases of duplicate transactions within block. +func LookupTransactionResultErrorMessagesByBlockIDUsingIndex(blockID flow.Identifier, txResultErrorMessages *[]flow.TransactionResultErrorMessage) func(*badger.Txn) error { + txErrIterFunc := func() (checkFunc, createFunc, handleFunc) { + check := func(_ []byte) bool { + return true + } + var val flow.TransactionResultErrorMessage + create := func() interface{} { + return &val + } + handle := func() error { + *txResultErrorMessages = append(*txResultErrorMessages, val) + return nil + } + return check, create, handle + } + + return traverse(makePrefix(codeTransactionResultErrorMessageIndex, blockID), txErrIterFunc) +} diff --git a/storage/badger/payloads_test.go b/storage/badger/payloads_test.go index cb11074f88b..d92a593526e 100644 --- a/storage/badger/payloads_test.go +++ b/storage/badger/payloads_test.go @@ -2,7 +2,6 @@ package badger_test import ( "errors" - "testing" "github.com/dgraph-io/badger/v2" diff --git a/storage/badger/transaction_result_error_messages.go b/storage/badger/transaction_result_error_messages.go new file mode 100644 index 00000000000..e2abf659d5e --- /dev/null +++ b/storage/badger/transaction_result_error_messages.go @@ -0,0 +1,212 @@ +package badger + +import ( + "fmt" + + "github.com/dgraph-io/badger/v2" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/storage/badger/operation" +) + +var _ storage.TransactionResultErrorMessages = (*TransactionResultErrorMessages)(nil) + +type TransactionResultErrorMessages struct { + db *badger.DB + cache *Cache[string, flow.TransactionResultErrorMessage] + indexCache *Cache[string, flow.TransactionResultErrorMessage] + blockCache *Cache[string, []flow.TransactionResultErrorMessage] +} + +func NewTransactionResultErrorMessages(collector module.CacheMetrics, db *badger.DB, transactionResultsCacheSize uint) *TransactionResultErrorMessages { + retrieve := func(key string) func(tx *badger.Txn) (flow.TransactionResultErrorMessage, error) { + var txResultErrMsg flow.TransactionResultErrorMessage + return func(tx *badger.Txn) (flow.TransactionResultErrorMessage, error) { + + blockID, txID, err := KeyToBlockIDTransactionID(key) + if err != nil { + return flow.TransactionResultErrorMessage{}, fmt.Errorf("could not convert key: %w", err) + } + + err = operation.RetrieveTransactionResultErrorMessage(blockID, txID, &txResultErrMsg)(tx) + if err != nil { + return flow.TransactionResultErrorMessage{}, handleError(err, flow.TransactionResultErrorMessage{}) + } + return txResultErrMsg, nil + } + } + retrieveIndex := func(key string) func(tx *badger.Txn) (flow.TransactionResultErrorMessage, error) { + var txResultErrMsg flow.TransactionResultErrorMessage + return func(tx *badger.Txn) (flow.TransactionResultErrorMessage, error) { + + blockID, txIndex, err := KeyToBlockIDIndex(key) + if err != nil { + return flow.TransactionResultErrorMessage{}, fmt.Errorf("could not convert index key: %w", err) + } + + err = operation.RetrieveTransactionResultErrorMessageByIndex(blockID, txIndex, &txResultErrMsg)(tx) + if err != nil { + return flow.TransactionResultErrorMessage{}, handleError(err, flow.TransactionResultErrorMessage{}) + } + return txResultErrMsg, nil + } + } + retrieveForBlock := func(key string) func(tx *badger.Txn) ([]flow.TransactionResultErrorMessage, error) { + var txResultErrMsg []flow.TransactionResultErrorMessage + return func(tx *badger.Txn) ([]flow.TransactionResultErrorMessage, error) { + + blockID, err := KeyToBlockID(key) + if err != nil { + return nil, fmt.Errorf("could not convert index key: %w", err) + } + + err = operation.LookupTransactionResultErrorMessagesByBlockIDUsingIndex(blockID, &txResultErrMsg)(tx) + if err != nil { + return nil, handleError(err, flow.TransactionResultErrorMessage{}) + } + return txResultErrMsg, nil + } + } + + return &TransactionResultErrorMessages{ + db: db, + cache: newCache[string, flow.TransactionResultErrorMessage](collector, metrics.ResourceTransactionResultErrorMessages, + withLimit[string, flow.TransactionResultErrorMessage](transactionResultsCacheSize), + withStore(noopStore[string, flow.TransactionResultErrorMessage]), + withRetrieve(retrieve), + ), + indexCache: newCache[string, flow.TransactionResultErrorMessage](collector, metrics.ResourceTransactionResultErrorMessagesIndices, + withLimit[string, flow.TransactionResultErrorMessage](transactionResultsCacheSize), + withStore(noopStore[string, flow.TransactionResultErrorMessage]), + withRetrieve(retrieveIndex), + ), + blockCache: newCache[string, []flow.TransactionResultErrorMessage](collector, metrics.ResourceTransactionResultErrorMessagesIndices, + withLimit[string, []flow.TransactionResultErrorMessage](transactionResultsCacheSize), + withStore(noopStore[string, []flow.TransactionResultErrorMessage]), + withRetrieve(retrieveForBlock), + ), + } +} + +// Store will store transaction result error messages for the given block ID. +// +// No errors are expected during normal operation. +func (t *TransactionResultErrorMessages) Store(blockID flow.Identifier, transactionResultErrorMessages []flow.TransactionResultErrorMessage) error { + batch := NewBatch(t.db) + + err := t.batchStore(blockID, transactionResultErrorMessages, batch) + if err != nil { + return err + } + + err = batch.Flush() + if err != nil { + return fmt.Errorf("cannot flush batch: %w", err) + } + + return nil +} + +// Exists returns true if transaction result error messages for the given ID have been stored. +// +// No errors are expected during normal operation. +func (t *TransactionResultErrorMessages) Exists(blockID flow.Identifier) (bool, error) { + // if the block is in the cache, return true + key := KeyFromBlockID(blockID) + if ok := t.blockCache.IsCached(key); ok { + return ok, nil + } + // otherwise, check badger store + var exists bool + err := t.db.View(operation.TransactionResultErrorMessagesExists(blockID, &exists)) + if err != nil { + return false, fmt.Errorf("could not check existence: %w", err) + } + return exists, nil +} + +// BatchStore inserts a batch of transaction result error messages into a batch +// +// No errors are expected during normal operation. +func (t *TransactionResultErrorMessages) batchStore( + blockID flow.Identifier, + transactionResultErrorMessages []flow.TransactionResultErrorMessage, + batch storage.BatchStorage, +) error { + writeBatch := batch.GetWriter() + + for _, result := range transactionResultErrorMessages { + err := operation.BatchInsertTransactionResultErrorMessage(blockID, &result)(writeBatch) + if err != nil { + return fmt.Errorf("cannot batch insert tx result error message: %w", err) + } + + err = operation.BatchIndexTransactionResultErrorMessage(blockID, &result)(writeBatch) + if err != nil { + return fmt.Errorf("cannot batch index tx result error message: %w", err) + } + } + + batch.OnSucceed(func() { + for _, result := range transactionResultErrorMessages { + key := KeyFromBlockIDTransactionID(blockID, result.TransactionID) + // cache for each transaction, so that it's faster to retrieve + t.cache.Insert(key, result) + + keyIndex := KeyFromBlockIDIndex(blockID, result.Index) + t.indexCache.Insert(keyIndex, result) + } + + key := KeyFromBlockID(blockID) + t.blockCache.Insert(key, transactionResultErrorMessages) + }) + return nil +} + +// ByBlockIDTransactionID returns the transaction result error message for the given block ID and transaction ID +// +// Expected errors during normal operation: +// - `storage.ErrNotFound` if no transaction error message is known at given block and transaction id. +func (t *TransactionResultErrorMessages) ByBlockIDTransactionID(blockID flow.Identifier, transactionID flow.Identifier) (*flow.TransactionResultErrorMessage, error) { + tx := t.db.NewTransaction(false) + defer tx.Discard() + key := KeyFromBlockIDTransactionID(blockID, transactionID) + transactionResultErrorMessage, err := t.cache.Get(key)(tx) + if err != nil { + return nil, err + } + return &transactionResultErrorMessage, nil +} + +// ByBlockIDTransactionIndex returns the transaction result error message for the given blockID and transaction index +// +// Expected errors during normal operation: +// - `storage.ErrNotFound` if no transaction error message is known at given block and transaction index. +func (t *TransactionResultErrorMessages) ByBlockIDTransactionIndex(blockID flow.Identifier, txIndex uint32) (*flow.TransactionResultErrorMessage, error) { + tx := t.db.NewTransaction(false) + defer tx.Discard() + key := KeyFromBlockIDIndex(blockID, txIndex) + transactionResultErrorMessage, err := t.indexCache.Get(key)(tx) + if err != nil { + return nil, err + } + return &transactionResultErrorMessage, nil +} + +// ByBlockID gets all transaction result error messages for a block, ordered by transaction index. +// Note: This method will return an empty slice both if the block is not indexed yet and if the block does not have any errors. +// +// No errors are expected during normal operation. +func (t *TransactionResultErrorMessages) ByBlockID(blockID flow.Identifier) ([]flow.TransactionResultErrorMessage, error) { + tx := t.db.NewTransaction(false) + defer tx.Discard() + key := KeyFromBlockID(blockID) + transactionResultErrorMessages, err := t.blockCache.Get(key)(tx) + if err != nil { + return nil, err + } + return transactionResultErrorMessages, nil +} diff --git a/storage/badger/transaction_result_error_messages_test.go b/storage/badger/transaction_result_error_messages_test.go new file mode 100644 index 00000000000..e21e8aaf348 --- /dev/null +++ b/storage/badger/transaction_result_error_messages_test.go @@ -0,0 +1,110 @@ +package badger_test + +import ( + "fmt" + "testing" + + "github.com/dgraph-io/badger/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/exp/rand" + + "github.com/onflow/flow-go/model/flow" + "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/storage" + "github.com/onflow/flow-go/utils/unittest" + + bstorage "github.com/onflow/flow-go/storage/badger" +) + +func TestStoringTransactionResultErrorMessages(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewTransactionResultErrorMessages(metrics, db, 1000) + + blockID := unittest.IdentifierFixture() + + // test db Exists by block id + exists, err := store.Exists(blockID) + require.NoError(t, err) + require.False(t, exists) + + // check retrieving by ByBlockID + messages, err := store.ByBlockID(blockID) + require.NoError(t, err) + require.Nil(t, messages) + + txErrorMessages := make([]flow.TransactionResultErrorMessage, 0) + for i := 0; i < 10; i++ { + expected := flow.TransactionResultErrorMessage{ + TransactionID: unittest.IdentifierFixture(), + ErrorMessage: fmt.Sprintf("a runtime error %d", i), + ExecutorID: unittest.IdentifierFixture(), + Index: rand.Uint32(), + } + txErrorMessages = append(txErrorMessages, expected) + } + err = store.Store(blockID, txErrorMessages) + require.NoError(t, err) + + // test db Exists by block id + exists, err = store.Exists(blockID) + require.NoError(t, err) + require.True(t, exists) + + // check retrieving by ByBlockIDTransactionID + for _, txErrorMessage := range txErrorMessages { + actual, err := store.ByBlockIDTransactionID(blockID, txErrorMessage.TransactionID) + require.NoError(t, err) + assert.Equal(t, txErrorMessage, *actual) + } + + // check retrieving by ByBlockIDTransactionIndex + for _, txErrorMessage := range txErrorMessages { + actual, err := store.ByBlockIDTransactionIndex(blockID, txErrorMessage.Index) + require.NoError(t, err) + assert.Equal(t, txErrorMessage, *actual) + } + + // check retrieving by ByBlockID + actual, err := store.ByBlockID(blockID) + require.NoError(t, err) + assert.Equal(t, txErrorMessages, actual) + + // test loading from database + newStore := bstorage.NewTransactionResultErrorMessages(metrics, db, 1000) + for _, txErrorMessage := range txErrorMessages { + actual, err := newStore.ByBlockIDTransactionID(blockID, txErrorMessage.TransactionID) + require.NoError(t, err) + assert.Equal(t, txErrorMessage, *actual) + } + + // check retrieving by index from both cache and db + for i, txErrorMessage := range txErrorMessages { + actual, err := store.ByBlockIDTransactionIndex(blockID, txErrorMessage.Index) + require.NoError(t, err) + assert.Equal(t, txErrorMessages[i], *actual) + + actual, err = newStore.ByBlockIDTransactionIndex(blockID, txErrorMessage.Index) + require.NoError(t, err) + assert.Equal(t, txErrorMessages[i], *actual) + } + }) +} + +func TestReadingNotStoreTransactionResultErrorMessage(t *testing.T) { + unittest.RunWithBadgerDB(t, func(db *badger.DB) { + metrics := metrics.NewNoopCollector() + store := bstorage.NewTransactionResultErrorMessages(metrics, db, 1000) + + blockID := unittest.IdentifierFixture() + txID := unittest.IdentifierFixture() + txIndex := rand.Uint32() + + _, err := store.ByBlockIDTransactionID(blockID, txID) + assert.ErrorIs(t, err, storage.ErrNotFound) + + _, err = store.ByBlockIDTransactionIndex(blockID, txIndex) + assert.ErrorIs(t, err, storage.ErrNotFound) + }) +} diff --git a/storage/mock/transaction_result_error_messages.go b/storage/mock/transaction_result_error_messages.go new file mode 100644 index 00000000000..c1bebf7f326 --- /dev/null +++ b/storage/mock/transaction_result_error_messages.go @@ -0,0 +1,163 @@ +// Code generated by mockery v2.43.2. DO NOT EDIT. + +package mock + +import ( + flow "github.com/onflow/flow-go/model/flow" + mock "github.com/stretchr/testify/mock" +) + +// TransactionResultErrorMessages is an autogenerated mock type for the TransactionResultErrorMessages type +type TransactionResultErrorMessages struct { + mock.Mock +} + +// ByBlockID provides a mock function with given fields: id +func (_m *TransactionResultErrorMessages) ByBlockID(id flow.Identifier) ([]flow.TransactionResultErrorMessage, error) { + ret := _m.Called(id) + + if len(ret) == 0 { + panic("no return value specified for ByBlockID") + } + + var r0 []flow.TransactionResultErrorMessage + var r1 error + if rf, ok := ret.Get(0).(func(flow.Identifier) ([]flow.TransactionResultErrorMessage, error)); ok { + return rf(id) + } + if rf, ok := ret.Get(0).(func(flow.Identifier) []flow.TransactionResultErrorMessage); ok { + r0 = rf(id) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]flow.TransactionResultErrorMessage) + } + } + + if rf, ok := ret.Get(1).(func(flow.Identifier) error); ok { + r1 = rf(id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ByBlockIDTransactionID provides a mock function with given fields: blockID, transactionID +func (_m *TransactionResultErrorMessages) ByBlockIDTransactionID(blockID flow.Identifier, transactionID flow.Identifier) (*flow.TransactionResultErrorMessage, error) { + ret := _m.Called(blockID, transactionID) + + if len(ret) == 0 { + panic("no return value specified for ByBlockIDTransactionID") + } + + var r0 *flow.TransactionResultErrorMessage + var r1 error + if rf, ok := ret.Get(0).(func(flow.Identifier, flow.Identifier) (*flow.TransactionResultErrorMessage, error)); ok { + return rf(blockID, transactionID) + } + if rf, ok := ret.Get(0).(func(flow.Identifier, flow.Identifier) *flow.TransactionResultErrorMessage); ok { + r0 = rf(blockID, transactionID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.TransactionResultErrorMessage) + } + } + + if rf, ok := ret.Get(1).(func(flow.Identifier, flow.Identifier) error); ok { + r1 = rf(blockID, transactionID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ByBlockIDTransactionIndex provides a mock function with given fields: blockID, txIndex +func (_m *TransactionResultErrorMessages) ByBlockIDTransactionIndex(blockID flow.Identifier, txIndex uint32) (*flow.TransactionResultErrorMessage, error) { + ret := _m.Called(blockID, txIndex) + + if len(ret) == 0 { + panic("no return value specified for ByBlockIDTransactionIndex") + } + + var r0 *flow.TransactionResultErrorMessage + var r1 error + if rf, ok := ret.Get(0).(func(flow.Identifier, uint32) (*flow.TransactionResultErrorMessage, error)); ok { + return rf(blockID, txIndex) + } + if rf, ok := ret.Get(0).(func(flow.Identifier, uint32) *flow.TransactionResultErrorMessage); ok { + r0 = rf(blockID, txIndex) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*flow.TransactionResultErrorMessage) + } + } + + if rf, ok := ret.Get(1).(func(flow.Identifier, uint32) error); ok { + r1 = rf(blockID, txIndex) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Exists provides a mock function with given fields: blockID +func (_m *TransactionResultErrorMessages) Exists(blockID flow.Identifier) (bool, error) { + ret := _m.Called(blockID) + + if len(ret) == 0 { + panic("no return value specified for Exists") + } + + var r0 bool + var r1 error + if rf, ok := ret.Get(0).(func(flow.Identifier) (bool, error)); ok { + return rf(blockID) + } + if rf, ok := ret.Get(0).(func(flow.Identifier) bool); ok { + r0 = rf(blockID) + } else { + r0 = ret.Get(0).(bool) + } + + if rf, ok := ret.Get(1).(func(flow.Identifier) error); ok { + r1 = rf(blockID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Store provides a mock function with given fields: blockID, transactionResultErrorMessages +func (_m *TransactionResultErrorMessages) Store(blockID flow.Identifier, transactionResultErrorMessages []flow.TransactionResultErrorMessage) error { + ret := _m.Called(blockID, transactionResultErrorMessages) + + if len(ret) == 0 { + panic("no return value specified for Store") + } + + var r0 error + if rf, ok := ret.Get(0).(func(flow.Identifier, []flow.TransactionResultErrorMessage) error); ok { + r0 = rf(blockID, transactionResultErrorMessages) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// NewTransactionResultErrorMessages creates a new instance of TransactionResultErrorMessages. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewTransactionResultErrorMessages(t interface { + mock.TestingT + Cleanup(func()) +}) *TransactionResultErrorMessages { + mock := &TransactionResultErrorMessages{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/storage/transaction_results.go b/storage/transaction_results.go index bb66023bc83..6aaacc3a880 100644 --- a/storage/transaction_results.go +++ b/storage/transaction_results.go @@ -33,3 +33,35 @@ type LightTransactionResults interface { // ByBlockID gets all transaction results for a block, ordered by transaction index ByBlockID(id flow.Identifier) ([]flow.LightTransactionResult, error) } + +// TransactionResultErrorMessages represents persistent storage for transaction result error messages +type TransactionResultErrorMessages interface { + + // Store will store transaction result error messages for the given block ID. + // + // No errors are expected during normal operation. + Store(blockID flow.Identifier, transactionResultErrorMessages []flow.TransactionResultErrorMessage) error + + // Exists returns true if transaction result error messages for the given ID have been stored. + // + // No errors are expected during normal operation. + Exists(blockID flow.Identifier) (bool, error) + + // ByBlockIDTransactionID returns the transaction result error message for the given block ID and transaction ID. + // + // Expected errors during normal operation: + // - `storage.ErrNotFound` if no transaction error message is known at given block and transaction id. + ByBlockIDTransactionID(blockID flow.Identifier, transactionID flow.Identifier) (*flow.TransactionResultErrorMessage, error) + + // ByBlockIDTransactionIndex returns the transaction result error message for the given blockID and transaction index. + // + // Expected errors during normal operation: + // - `storage.ErrNotFound` if no transaction error message is known at given block and transaction index. + ByBlockIDTransactionIndex(blockID flow.Identifier, txIndex uint32) (*flow.TransactionResultErrorMessage, error) + + // ByBlockID gets all transaction result error messages for a block, ordered by transaction index. + // Note: This method will return an empty slice both if the block is not indexed yet and if the block does not have any errors. + // + // No errors are expected during normal operation. + ByBlockID(id flow.Identifier) ([]flow.TransactionResultErrorMessage, error) +} diff --git a/utils/unittest/execution_state.go b/utils/unittest/execution_state.go index 425527670dc..a5e911b3771 100644 --- a/utils/unittest/execution_state.go +++ b/utils/unittest/execution_state.go @@ -23,7 +23,7 @@ const ServiceAccountPrivateKeySignAlgo = crypto.ECDSAP256 const ServiceAccountPrivateKeyHashAlgo = hash.SHA2_256 // Pre-calculated state commitment with root account with the above private key -const GenesisStateCommitmentHex = "d2a7db1fa0cc1eee36e8769c11095fbaa440ab7916e5923afa10e760fd5eae2b" +const GenesisStateCommitmentHex = "c42fc978c2702793d2640e3ed8644ba54db4e92aa5d0501234dfbb9bbc5784fd" var GenesisStateCommitment flow.StateCommitment @@ -87,10 +87,10 @@ func genesisCommitHexByChainID(chainID flow.ChainID) string { return GenesisStateCommitmentHex } if chainID == flow.Testnet { - return "719c12f8e28a40d822e43873d8a188daa7a3f81dc530c4c61f4141d76985bd46" + return "e29456decb9ee90ad3ed1e1239383c18897b031ea851ff07f5f616657df4d4a0" } if chainID == flow.Sandboxnet { return "e1c08b17f9e5896f03fe28dd37ca396c19b26628161506924fbf785834646ea1" } - return "5efd4c97fa23bd76769ab891a2681a839f8bdd0a2be0e07ab841f51b2e3a2f51" + return "e1989abf50fba23015251a313eefe2ceff45639a75252f4da5970dcda32dd95e" } diff --git a/utils/unittest/fixtures.go b/utils/unittest/fixtures.go index 0bce5e99b41..9c3f9784493 100644 --- a/utils/unittest/fixtures.go +++ b/utils/unittest/fixtures.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/require" sdk "github.com/onflow/flow-go-sdk" + hotstuff "github.com/onflow/flow-go/consensus/hotstuff/model" "github.com/onflow/flow-go/engine" "github.com/onflow/flow-go/engine/access/rest/util"