diff --git a/core/gethwrappers/generated/i_keeper_registry_master_wrapper_2_1/i_keeper_registry_master_wrapper_2_1.go b/core/gethwrappers/generated/i_keeper_registry_master_wrapper_2_1/i_keeper_registry_master_wrapper_2_1.go index 640e871c084..a79d0c9c876 100644 --- a/core/gethwrappers/generated/i_keeper_registry_master_wrapper_2_1/i_keeper_registry_master_wrapper_2_1.go +++ b/core/gethwrappers/generated/i_keeper_registry_master_wrapper_2_1/i_keeper_registry_master_wrapper_2_1.go @@ -196,9 +196,7 @@ func (_IKeeperRegistryMaster *IKeeperRegistryMasterTransactorRaw) Transact(opts return _IKeeperRegistryMaster.Contract.contract.Transact(opts, method, params...) } -func (_IKeeperRegistryMaster *IKeeperRegistryMasterCaller) CheckCallback(opts *bind.CallOpts, id *big.Int, values [][]byte, extraData []byte) (CheckCallback, - - error) { +func (_IKeeperRegistryMaster *IKeeperRegistryMasterCaller) CheckCallback(opts *bind.CallOpts, id *big.Int, values [][]byte, extraData []byte) (CheckCallback, error) { var out []interface{} err := _IKeeperRegistryMaster.contract.Call(opts, &out, "checkCallback", id, values, extraData) diff --git a/core/scripts/chaincli/handler/debug.go b/core/scripts/chaincli/handler/debug.go index daf012ee16e..b2f5c633af0 100644 --- a/core/scripts/chaincli/handler/debug.go +++ b/core/scripts/chaincli/handler/debug.go @@ -18,16 +18,21 @@ import ( gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/chainlink/core/scripts/chaincli/config" "github.com/smartcontractkit/chainlink/core/scripts/common" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_utils_2_1" iregistry21 "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/i_keeper_registry_master_wrapper_2_1" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/models" evm "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21" "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/core" "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/encoding" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/streams" "github.com/smartcontractkit/chainlink/v2/core/utils" bigmath "github.com/smartcontractkit/chainlink/v2/core/utils/big_math" - ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" ) const ( @@ -227,6 +232,13 @@ func (k *Keeper) Debug(ctx context.Context, args []string) { message(fmt.Sprintf("checkUpkeep failed with UpkeepFailureReason %d", checkResult.UpkeepFailureReason)) } if checkResult.UpkeepFailureReason == uint8(encoding.UpkeepFailureReasonTargetCheckReverted) { + mc := &models.MercuryCredentials{k.cfg.MercuryLegacyURL, k.cfg.MercuryURL, k.cfg.MercuryID, k.cfg.MercuryKey} + mercuryConfig := evm.NewMercuryConfig(mc, core.StreamsCompatibleABI) + lggr, _ := logger.NewLogger() + blockSub := &blockSubscriber{k.client} + + _ = streams.NewStreamsLookup(packer, mercuryConfig, blockSub, keeperRegistry21, lggr) + streamsLookupErr, err := packer.DecodeStreamsLookupRequest(checkResult.PerformData) if err == nil { message("upkeep reverted with StreamsLookup") @@ -240,7 +252,7 @@ func (k *Keeper) Debug(ctx context.Context, args []string) { } allowed := false if len(cfg) > 0 { - var privilegeConfig evm.UpkeepPrivilegeConfig + var privilegeConfig streams.UpkeepPrivilegeConfig if err := json.Unmarshal(cfg, &privilegeConfig); err != nil { failUnknown("failed to unmarshal privilege config ", err) } @@ -307,6 +319,22 @@ func (k *Keeper) Debug(ctx context.Context, args []string) { } } +type blockSubscriber struct { + ethClient *ethclient.Client +} + +func (bs *blockSubscriber) LatestBlock() *ocr2keepers.BlockKey { + header, err := bs.ethClient.HeaderByNumber(context.Background(), nil) + if err != nil { + return nil + } + + return &ocr2keepers.BlockKey{ + Number: ocr2keepers.BlockNumber(header.Number.Uint64()), + Hash: header.Hash(), + } +} + func logMatchesTriggerConfig(log *types.Log, config automation_utils_2_1.LogTriggerConfig) bool { if log.Topics[0] != config.Topic0 { return false diff --git a/core/scripts/chaincli/handler/keeper.go b/core/scripts/chaincli/handler/keeper.go index ba8276efb56..453ef43a566 100644 --- a/core/scripts/chaincli/handler/keeper.go +++ b/core/scripts/chaincli/handler/keeper.go @@ -14,6 +14,8 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/umbracle/ethgo/abi" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/streams" + "github.com/smartcontractkit/chainlink/core/scripts/chaincli/config" helpers "github.com/smartcontractkit/chainlink/core/scripts/common" "github.com/smartcontractkit/chainlink/v2/core/cmd" @@ -34,7 +36,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/verifiable_load_upkeep_wrapper" "github.com/smartcontractkit/chainlink/v2/core/logger" "github.com/smartcontractkit/chainlink/v2/core/services/keeper" - evm "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21" ) // Keeper is the keepers commands handler @@ -719,7 +720,7 @@ func (k *Keeper) deployUpkeeps(ctx context.Context, registryAddr common.Address, log.Printf("registry version is %s", v) log.Printf("active upkeep ids: %v", activeUpkeepIds) - adminBytes, err := json.Marshal(evm.UpkeepPrivilegeConfig{ + adminBytes, err := json.Marshal(streams.UpkeepPrivilegeConfig{ MercuryEnabled: true, }) if err != nil { diff --git a/core/scripts/chaincli/handler/mercury_lookup_handler.go b/core/scripts/chaincli/handler/mercury_lookup_handler.go index 0db142df9f9..1bd4b2e183c 100644 --- a/core/scripts/chaincli/handler/mercury_lookup_handler.go +++ b/core/scripts/chaincli/handler/mercury_lookup_handler.go @@ -3,7 +3,9 @@ package handler import ( "context" "crypto/hmac" + "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io" "math/big" @@ -13,10 +15,6 @@ import ( "strings" "time" - "crypto/sha256" - - "encoding/json" - "github.com/avast/retry-go" ethabi "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/block_subscriber.go b/core/services/ocr2/plugins/ocr2keeper/evm21/block_subscriber.go index d97156ed180..6cc19a4d02e 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evm21/block_subscriber.go +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/block_subscriber.go @@ -12,6 +12,7 @@ import ( ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" "github.com/smartcontractkit/chainlink-relay/pkg/services" + httypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker/types" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller" evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" @@ -56,6 +57,10 @@ type BlockSubscriber struct { lggr logger.Logger } +func (bs *BlockSubscriber) LatestBlock() *ocr2keepers.BlockKey { + return bs.latestBlock.Load() +} + var _ ocr2keepers.BlockSubscriber = &BlockSubscriber{} func NewBlockSubscriber(hb httypes.HeadBroadcaster, lp logpoller.LogPoller, finalityDepth uint32, lggr logger.Logger) *BlockSubscriber { diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/encoding/interface.go b/core/services/ocr2/plugins/ocr2keeper/evm21/encoding/interface.go index 217f0dcb571..e21ecfb800d 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evm21/encoding/interface.go +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/encoding/interface.go @@ -5,56 +5,55 @@ import ( ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_utils_2_1" iregistry21 "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/i_keeper_registry_master_wrapper_2_1" ) -type UpkeepFailureReason uint8 -type PipelineExecutionState uint8 - const ( // upkeep failure onchain reasons - UpkeepFailureReasonNone UpkeepFailureReason = 0 - UpkeepFailureReasonUpkeepCancelled UpkeepFailureReason = 1 - UpkeepFailureReasonUpkeepPaused UpkeepFailureReason = 2 - UpkeepFailureReasonTargetCheckReverted UpkeepFailureReason = 3 - UpkeepFailureReasonUpkeepNotNeeded UpkeepFailureReason = 4 - UpkeepFailureReasonPerformDataExceedsLimit UpkeepFailureReason = 5 - UpkeepFailureReasonInsufficientBalance UpkeepFailureReason = 6 - UpkeepFailureReasonMercuryCallbackReverted UpkeepFailureReason = 7 - UpkeepFailureReasonRevertDataExceedsLimit UpkeepFailureReason = 8 - UpkeepFailureReasonRegistryPaused UpkeepFailureReason = 9 + UpkeepFailureReasonNone uint8 = 0 + UpkeepFailureReasonUpkeepCancelled uint8 = 1 + UpkeepFailureReasonUpkeepPaused uint8 = 2 + UpkeepFailureReasonTargetCheckReverted uint8 = 3 + UpkeepFailureReasonUpkeepNotNeeded uint8 = 4 + UpkeepFailureReasonPerformDataExceedsLimit uint8 = 5 + UpkeepFailureReasonInsufficientBalance uint8 = 6 + UpkeepFailureReasonMercuryCallbackReverted uint8 = 7 + UpkeepFailureReasonRevertDataExceedsLimit uint8 = 8 + UpkeepFailureReasonRegistryPaused uint8 = 9 // leaving a gap here for more onchain failure reasons in the future // upkeep failure offchain reasons - UpkeepFailureReasonMercuryAccessNotAllowed UpkeepFailureReason = 32 - UpkeepFailureReasonTxHashNoLongerExists UpkeepFailureReason = 33 - UpkeepFailureReasonInvalidRevertDataInput UpkeepFailureReason = 34 - UpkeepFailureReasonSimulationFailed UpkeepFailureReason = 35 - UpkeepFailureReasonTxHashReorged UpkeepFailureReason = 36 + UpkeepFailureReasonMercuryAccessNotAllowed uint8 = 32 + UpkeepFailureReasonTxHashNoLongerExists uint8 = 33 + UpkeepFailureReasonInvalidRevertDataInput uint8 = 34 + UpkeepFailureReasonSimulationFailed uint8 = 35 + UpkeepFailureReasonTxHashReorged uint8 = 36 // pipeline execution error - NoPipelineError PipelineExecutionState = 0 - CheckBlockTooOld PipelineExecutionState = 1 - CheckBlockInvalid PipelineExecutionState = 2 - RpcFlakyFailure PipelineExecutionState = 3 - MercuryFlakyFailure PipelineExecutionState = 4 - PackUnpackDecodeFailed PipelineExecutionState = 5 - MercuryUnmarshalError PipelineExecutionState = 6 - InvalidMercuryRequest PipelineExecutionState = 7 - InvalidMercuryResponse PipelineExecutionState = 8 // this will only happen if Mercury server sends bad responses - UpkeepNotAuthorized PipelineExecutionState = 9 + NoPipelineError uint8 = 0 + CheckBlockTooOld uint8 = 1 + CheckBlockInvalid uint8 = 2 + RpcFlakyFailure uint8 = 3 + MercuryFlakyFailure uint8 = 4 + PackUnpackDecodeFailed uint8 = 5 + MercuryUnmarshalError uint8 = 6 + InvalidMercuryRequest uint8 = 7 + InvalidMercuryResponse uint8 = 8 // this will only happen if Mercury server sends bad responses + UpkeepNotAuthorized uint8 = 9 ) type UpkeepInfo = iregistry21.KeeperRegistryBase21UpkeepInfo type Packer interface { UnpackCheckResult(payload ocr2keepers.UpkeepPayload, raw string) (ocr2keepers.CheckResult, error) - UnpackCheckCallbackResult(callbackResp []byte) (PipelineExecutionState, bool, []byte, uint8, *big.Int, error) - UnpackPerformResult(raw string) (PipelineExecutionState, bool, error) + UnpackCheckCallbackResult(callbackResp []byte) (uint8, bool, []byte, uint8, *big.Int, error) + UnpackPerformResult(raw string) (uint8, bool, error) UnpackLogTriggerConfig(raw []byte) (automation_utils_2_1.LogTriggerConfig, error) PackReport(report automation_utils_2_1.KeeperRegistryBase21Report) ([]byte, error) UnpackReport(raw []byte) (automation_utils_2_1.KeeperRegistryBase21Report, error) PackGetUpkeepPrivilegeConfig(upkeepId *big.Int) ([]byte, error) UnpackGetUpkeepPrivilegeConfig(resp []byte) ([]byte, error) - DecodeStreamsLookupRequest(data []byte) (*StreamsLookupError, error) + DecodeStreamsLookupRequest(data []byte) (*mercury.StreamsLookupError, error) } diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/encoding/packer.go b/core/services/ocr2/plugins/ocr2keeper/evm21/encoding/packer.go index 9e070b2838c..28c6c4a9759 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evm21/encoding/packer.go +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/encoding/packer.go @@ -6,8 +6,11 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common/hexutil" + ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_utils_2_1" "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/core" ) @@ -80,7 +83,7 @@ func (p *abiPacker) UnpackGetUpkeepPrivilegeConfig(resp []byte) ([]byte, error) return bts, nil } -func (p *abiPacker) UnpackCheckCallbackResult(callbackResp []byte) (PipelineExecutionState, bool, []byte, uint8, *big.Int, error) { +func (p *abiPacker) UnpackCheckCallbackResult(callbackResp []byte) (uint8, bool, []byte, uint8, *big.Int, error) { out, err := p.registryABI.Methods["checkCallback"].Outputs.UnpackValues(callbackResp) if err != nil { return PackUnpackDecodeFailed, false, nil, 0, nil, fmt.Errorf("%w: unpack checkUpkeep return: %s", err, hexutil.Encode(callbackResp)) @@ -94,7 +97,7 @@ func (p *abiPacker) UnpackCheckCallbackResult(callbackResp []byte) (PipelineExec return NoPipelineError, upkeepNeeded, rawPerformData, failureReason, gasUsed, nil } -func (p *abiPacker) UnpackPerformResult(raw string) (PipelineExecutionState, bool, error) { +func (p *abiPacker) UnpackPerformResult(raw string) (uint8, bool, error) { b, err := hexutil.Decode(raw) if err != nil { return PackUnpackDecodeFailed, false, err @@ -161,16 +164,8 @@ func (p *abiPacker) UnpackReport(raw []byte) (automation_utils_2_1.KeeperRegistr return report, nil } -type StreamsLookupError struct { - FeedParamKey string - Feeds []string - TimeParamKey string - Time *big.Int - ExtraData []byte -} - // DecodeStreamsLookupRequest decodes the revert error StreamsLookup(string feedParamKey, string[] feeds, string feedParamKey, uint256 time, byte[] extraData) -func (p *abiPacker) DecodeStreamsLookupRequest(data []byte) (*StreamsLookupError, error) { +func (p *abiPacker) DecodeStreamsLookupRequest(data []byte) (*mercury.StreamsLookupError, error) { e := p.streamsABI.Errors["StreamsLookup"] unpack, err := e.Unpack(data) if err != nil { @@ -178,7 +173,7 @@ func (p *abiPacker) DecodeStreamsLookupRequest(data []byte) (*StreamsLookupError } errorParameters := unpack.([]interface{}) - return &StreamsLookupError{ + return &mercury.StreamsLookupError{ FeedParamKey: *abi.ConvertType(errorParameters[0], new(string)).(*string), Feeds: *abi.ConvertType(errorParameters[1], new([]string)).(*[]string), TimeParamKey: *abi.ConvertType(errorParameters[2], new(string)).(*string), @@ -188,10 +183,10 @@ func (p *abiPacker) DecodeStreamsLookupRequest(data []byte) (*StreamsLookupError } // GetIneligibleCheckResultWithoutPerformData returns an ineligible check result with ineligibility reason and pipeline execution state but without perform data -func GetIneligibleCheckResultWithoutPerformData(p ocr2keepers.UpkeepPayload, reason UpkeepFailureReason, state PipelineExecutionState, retryable bool) ocr2keepers.CheckResult { +func GetIneligibleCheckResultWithoutPerformData(p ocr2keepers.UpkeepPayload, reason uint8, state uint8, retryable bool) ocr2keepers.CheckResult { return ocr2keepers.CheckResult{ - IneligibilityReason: uint8(reason), - PipelineExecutionState: uint8(state), + IneligibilityReason: reason, + PipelineExecutionState: state, Retryable: retryable, UpkeepID: p.UpkeepID, Trigger: p.Trigger, diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/encoding/packer_test.go b/core/services/ocr2/plugins/ocr2keeper/evm21/encoding/packer_test.go index 1801b018572..03b55bfccc7 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evm21/encoding/packer_test.go +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/encoding/packer_test.go @@ -13,6 +13,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_utils_2_1" automation21Utils "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_utils_2_1" "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/core" @@ -210,7 +212,7 @@ func TestPacker_UnpackPerformResult(t *testing.T) { tests := []struct { Name string RawData string - State PipelineExecutionState + State uint8 }{ { Name: "unpack success", @@ -238,7 +240,7 @@ func TestPacker_UnpackCheckCallbackResult(t *testing.T) { FailureReason uint8 GasUsed *big.Int ErrorString string - State PipelineExecutionState + State uint8 }{ { Name: "unpack upkeep needed", @@ -441,14 +443,14 @@ func TestPacker_DecodeStreamsLookupRequest(t *testing.T) { tests := []struct { name string data []byte - expected *StreamsLookupError - state PipelineExecutionState + expected *mercury.StreamsLookupError + state uint8 err error }{ { name: "success - decode to streams lookup", data: hexutil.MustDecode("0xf055e4a200000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002400000000000000000000000000000000000000000000000000000000002435eb50000000000000000000000000000000000000000000000000000000000000280000000000000000000000000000000000000000000000000000000000000000966656564496448657800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000423078343535343438326435353533343432643431353234323439353435323535346432643534343535333534346534353534303030303030303030303030303030300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000042307834323534343332643535353334343264343135323432343935343532353534643264353434353533353434653435353430303030303030303030303030303030000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000b626c6f636b4e756d62657200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000064000000000000000000000000"), - expected: &StreamsLookupError{ + expected: &mercury.StreamsLookupError{ FeedParamKey: "feedIdHex", Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, TimeParamKey: "blockNumber", diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/mercury.go b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/mercury.go new file mode 100644 index 00000000000..9d340ef26fd --- /dev/null +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/mercury.go @@ -0,0 +1,122 @@ +package mercury + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "fmt" + "math/big" + "net/http" + "time" + + "github.com/patrickmn/go-cache" + + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/models" +) + +const ( + FeedIDs = "feedIDs" // valid for v0.3 + FeedIdHex = "feedIdHex" // valid for v0.2 + BlockNumber = "blockNumber" // valid for v0.2 + Timestamp = "timestamp" // valid for v0.3 + totalFastPluginRetries = 5 + totalMediumPluginRetries = 10 +) + +var GenerateHMACFn = func(method string, path string, body []byte, clientId string, secret string, ts int64) string { + bodyHash := sha256.New() + bodyHash.Write(body) + hashString := fmt.Sprintf("%s %s %s %s %d", + method, + path, + hex.EncodeToString(bodyHash.Sum(nil)), + clientId, + ts) + signedMessage := hmac.New(sha256.New, []byte(secret)) + signedMessage.Write([]byte(hashString)) + userHmac := hex.EncodeToString(signedMessage.Sum(nil)) + return userHmac +} + +// CalculateRetryConfig returns plugin retry interval based on how many times plugin has retried this work +var CalculateRetryConfigFn = func(prk string, mercuryConfig MercuryConfigProvider) time.Duration { + var ri time.Duration + var retries int + totalAttempts, ok := mercuryConfig.GetPluginRetry(prk) + if ok { + retries = totalAttempts.(int) + if retries < totalFastPluginRetries { + ri = 1 * time.Second + } else if retries < totalMediumPluginRetries { + ri = 5 * time.Second + } + // if the core node has retried totalMediumPluginRetries times, do not set retry interval and plugin will use + // the default interval + } else { + ri = 1 * time.Second + } + mercuryConfig.SetPluginRetry(prk, retries+1, cache.DefaultExpiration) + return ri +} + +type MercuryData struct { + Index int + Error error + Retryable bool + Bytes [][]byte + State MercuryUpkeepState +} + +type Packer interface { + UnpackCheckCallbackResult(callbackResp []byte) (uint8, bool, []byte, uint8, *big.Int, error) + PackGetUpkeepPrivilegeConfig(upkeepId *big.Int) ([]byte, error) + UnpackGetUpkeepPrivilegeConfig(resp []byte) ([]byte, error) + DecodeStreamsLookupRequest(data []byte) (*StreamsLookupError, error) +} + +type MercuryConfigProvider interface { + Credentials() *models.MercuryCredentials + IsUpkeepAllowed(string) (interface{}, bool) + SetUpkeepAllowed(string, interface{}, time.Duration) + GetPluginRetry(string) (interface{}, bool) + SetPluginRetry(string, interface{}, time.Duration) +} + +type HttpClient interface { + Do(req *http.Request) (*http.Response, error) +} + +type MercuryClient interface { + DoRequest(ctx context.Context, streamsLookup *StreamsLookup, pluginRetryKey string) (MercuryUpkeepState, MercuryUpkeepFailureReason, [][]byte, bool, time.Duration, error) +} + +type StreamsLookupError struct { + FeedParamKey string + Feeds []string + TimeParamKey string + Time *big.Int + ExtraData []byte +} + +type StreamsLookup struct { + *StreamsLookupError + UpkeepId *big.Int + Block uint64 +} + +func (l *StreamsLookup) IsMercuryVersionUnkown() bool { + return l.FeedParamKey != FeedIDs +} + +func (l *StreamsLookup) IsMercuryV02() bool { + return l.FeedParamKey == FeedIdHex && l.TimeParamKey == BlockNumber +} + +func (l *StreamsLookup) IsMercuryV03() bool { + return l.FeedParamKey == FeedIDs +} + +func (l *StreamsLookup) IsMercuryUsingBatchPathV03() bool { + return l.TimeParamKey == BlockNumber +} diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/mercury_test.go b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/mercury_test.go new file mode 100644 index 00000000000..695575d8391 --- /dev/null +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/mercury_test.go @@ -0,0 +1,37 @@ +package mercury + +import ( + "testing" +) + +func TestGenerateHMACFn(t *testing.T) { + testCases := []struct { + method string + path string + body []byte + clientId string + secret string + ts int64 + expected string + }{ + { + method: "GET", + path: "/example", + body: []byte(""), + clientId: "yourClientId", + secret: "yourSecret", + ts: 1234567890, + expected: "17b0bb6b14f7b48ef9d24f941ff8f33ad2d5e94ac343380be02c2f1ca32fdbd8", + }, + } + + for _, tc := range testCases { + t.Run("", func(t *testing.T) { + result := GenerateHMACFn(tc.method, tc.path, tc.body, tc.clientId, tc.secret, tc.ts) + + if result != tc.expected { + t.Errorf("Expected: %s, Got: %s", tc.expected, result) + } + }) + } +} diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/streams/streams.go b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/streams/streams.go new file mode 100644 index 00000000000..61b99ac20ab --- /dev/null +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/streams/streams.go @@ -0,0 +1,271 @@ +package streams + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "net/http" + "sync" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/patrickmn/go-cache" + ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + + iregistry21 "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/i_keeper_registry_master_wrapper_2_1" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/core" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury" + v02 "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v02" + v03 "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v03" +) + +type Lookup interface { + Lookup(ctx context.Context, checkResults []ocr2keepers.CheckResult) []ocr2keepers.CheckResult +} + +type latestBlockProvider interface { + LatestBlock() *ocr2keepers.BlockKey +} + +type contextCaller interface { + CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error +} + +type streamsRegistry interface { + GetUpkeepPrivilegeConfig(opts *bind.CallOpts, upkeepId *big.Int) ([]byte, error) + CheckCallback(opts *bind.CallOpts, id *big.Int, values [][]byte, extraData []byte) (iregistry21.CheckCallback, error) +} + +type streams struct { + packer mercury.Packer + mercuryConfig mercury.MercuryConfigProvider + abi abi.ABI + blockSubscriber latestBlockProvider + registry streamsRegistry + lggr logger.Logger + v02Client mercury.MercuryClient + v03Client mercury.MercuryClient +} + +// UpkeepPrivilegeConfig represents the administrative offchain config for each upkeep. It can be set by s_upkeepPrivilegeManager +// role on the registry. Upkeeps allowed to use Mercury server will have this set to true. +type UpkeepPrivilegeConfig struct { + MercuryEnabled bool `json:"mercuryEnabled"` +} + +func NewStreamsLookup( + packer mercury.Packer, + mercuryConfig mercury.MercuryConfigProvider, + blockSubscriber latestBlockProvider, + registry streamsRegistry, + lggr logger.Logger) *streams { + httpClient := http.DefaultClient + return &streams{ + packer: packer, + mercuryConfig: mercuryConfig, + abi: core.RegistryABI, + blockSubscriber: blockSubscriber, + registry: registry, + lggr: lggr, + v02Client: v02.NewClient(mercuryConfig, httpClient, lggr), + v03Client: v03.NewClient(mercuryConfig, httpClient, lggr), + } +} + +// Lookup looks through check upkeep results looking for any that need off chain lookup +func (s *streams) Lookup(ctx context.Context, checkResults []ocr2keepers.CheckResult) []ocr2keepers.CheckResult { + lookups := map[int]*mercury.StreamsLookup{} + for i, checkResult := range checkResults { + s.buildResult(ctx, i, checkResult, checkResults, lookups) + } + + var wg sync.WaitGroup + for i, lookup := range lookups { + wg.Add(1) + go s.doLookup(ctx, &wg, lookup, i, checkResults) + } + wg.Wait() + + // don't surface error to plugin bc StreamsLookup process should be self-contained. + return checkResults +} + +// buildResult checks if the upkeep is allowed by Mercury and builds a streams lookup request from the check result +func (s *streams) buildResult(ctx context.Context, i int, checkResult ocr2keepers.CheckResult, checkResults []ocr2keepers.CheckResult, lookups map[int]*mercury.StreamsLookup) { + lookupLggr := s.lggr.With("where", "StreamsLookup") + if checkResult.IneligibilityReason != uint8(mercury.MercuryUpkeepFailureReasonTargetCheckReverted) { + // Streams Lookup only works when upkeep target check reverts + return + } + + block := big.NewInt(int64(checkResult.Trigger.BlockNumber)) + upkeepId := checkResult.UpkeepID + + if s.mercuryConfig.Credentials() == nil { + lookupLggr.Errorf("at block %d upkeep %s tries to access mercury server but mercury credential is not configured", block, upkeepId) + return + } + + // Try to decode the revert error into streams lookup format. User upkeeps can revert with any reason, see if they + // tried to call mercury + lookupLggr.Infof("at block %d upkeep %s trying to DecodeStreamsLookupRequest performData=%s", block, upkeepId, hexutil.Encode(checkResults[i].PerformData)) + streamsLookupErr, err := s.packer.DecodeStreamsLookupRequest(checkResult.PerformData) + if err != nil { + lookupLggr.Debugf("at block %d upkeep %s DecodeStreamsLookupRequest failed: %v", block, upkeepId, err) + // user contract did not revert with StreamsLookup error + return + } + streamsLookupResponse := &mercury.StreamsLookup{StreamsLookupError: streamsLookupErr} + + if len(streamsLookupResponse.Feeds) == 0 { + checkResults[i].IneligibilityReason = uint8(mercury.MercuryUpkeepFailureReasonInvalidRevertDataInput) + lookupLggr.Debugf("at block %s upkeep %s has empty feeds array", block, upkeepId) + return + } + + // mercury permission checking for v0.3 is done by mercury server + if streamsLookupResponse.IsMercuryV02() { + // check permission on the registry for mercury v0.2 + opts := s.buildCallOpts(ctx, block) + if state, reason, retryable, allowed, err := s.allowedToUseMercury(opts, upkeepId.BigInt()); err != nil { + lookupLggr.Warnf("at block %s upkeep %s failed to query mercury allow list: %s", block, upkeepId, err) + checkResults[i].PipelineExecutionState = uint8(state) + checkResults[i].IneligibilityReason = uint8(reason) + checkResults[i].Retryable = retryable + return + } else if !allowed { + lookupLggr.Debugf("at block %d upkeep %s NOT allowed to query Mercury server", block, upkeepId) + checkResults[i].IneligibilityReason = uint8(mercury.MercuryUpkeepFailureReasonMercuryAccessNotAllowed) + return + } + } else if streamsLookupResponse.IsMercuryVersionUnkown() { + // if mercury version cannot be determined, set failure reason + lookupLggr.Debugf("at block %d upkeep %s NOT allowed to query Mercury server", block, upkeepId) + checkResults[i].IneligibilityReason = uint8(mercury.MercuryUpkeepFailureReasonInvalidRevertDataInput) + return + } + + streamsLookupResponse.UpkeepId = upkeepId.BigInt() + // the block here is exclusively used to call checkCallback at this block, not to be confused with the block number + // in the revert for mercury v0.2, which is denoted by time in the struct bc starting from v0.3, only timestamp will be supported + streamsLookupResponse.Block = uint64(block.Int64()) + lookupLggr.Infof("at block %d upkeep %s DecodeStreamsLookupRequest feedKey=%s timeKey=%s feeds=%v time=%s extraData=%s", block, upkeepId, streamsLookupResponse.FeedParamKey, streamsLookupResponse.TimeParamKey, streamsLookupResponse.Feeds, streamsLookupResponse.Time, hexutil.Encode(streamsLookupResponse.ExtraData)) + lookups[i] = streamsLookupResponse +} + +func (s *streams) doLookup(ctx context.Context, wg *sync.WaitGroup, lookup *mercury.StreamsLookup, i int, checkResults []ocr2keepers.CheckResult) { + defer wg.Done() + + state, reason, values, retryable, retryInterval, err := mercury.NoPipelineError, mercury.MercuryUpkeepFailureReasonInvalidRevertDataInput, [][]byte{}, false, 0*time.Second, fmt.Errorf("invalid revert data input: feed param key %s, time param key %s, feeds %s", lookup.FeedParamKey, lookup.TimeParamKey, lookup.Feeds) + pluginRetryKey := generatePluginRetryKey(checkResults[i].WorkID, lookup.Block) + + if lookup.IsMercuryV02() { + state, reason, values, retryable, retryInterval, err = s.v02Client.DoRequest(ctx, lookup, pluginRetryKey) + } else if lookup.IsMercuryV03() { + state, reason, values, retryable, retryInterval, err = s.v03Client.DoRequest(ctx, lookup, pluginRetryKey) + } + + if err != nil { + s.lggr.Errorf("upkeep %s retryable %v retryInterval %s doMercuryRequest: %s", lookup.UpkeepId, retryable, retryInterval, err.Error()) + checkResults[i].Retryable = retryable + checkResults[i].RetryInterval = retryInterval + checkResults[i].PipelineExecutionState = uint8(state) + checkResults[i].IneligibilityReason = uint8(reason) + return + } + + for j, v := range values { + s.lggr.Infof("upkeep %s doMercuryRequest values[%d]: %s", lookup.UpkeepId, j, hexutil.Encode(v)) + } + + state, retryable, checkCallbackResult, err := s.checkCallback(ctx, values, lookup) + if err != nil { + s.lggr.Errorf("at block %d upkeep %s checkCallback err: %s", lookup.Block, lookup.UpkeepId, err.Error()) + checkResults[i].Retryable = retryable + checkResults[i].PipelineExecutionState = uint8(state) + return + } + s.lggr.Infof("checkCallback mercuryBytes=%+v", checkCallbackResult) + + if checkCallbackResult.UpkeepFailureReason == uint8(mercury.MercuryUpkeepFailureReasonMercuryCallbackReverted) { + checkResults[i].IneligibilityReason = uint8(mercury.MercuryUpkeepFailureReasonMercuryCallbackReverted) + s.lggr.Debugf("at block %d upkeep %s mercury callback reverts", lookup.Block, lookup.UpkeepId) + return + } + + if !checkCallbackResult.UpkeepNeeded { + checkResults[i].IneligibilityReason = uint8(mercury.MercuryUpkeepFailureReasonUpkeepNotNeeded) + s.lggr.Debugf("at block %d upkeep %s callback reports upkeep not needed", lookup.Block, lookup.UpkeepId) + return + } + + checkResults[i].IneligibilityReason = uint8(mercury.MercuryUpkeepFailureReasonNone) + checkResults[i].Eligible = true + checkResults[i].PerformData = checkCallbackResult.PerformData + s.lggr.Infof("at block %d upkeep %s successful with perform data: %s", lookup.Block, lookup.UpkeepId, hexutil.Encode(checkCallbackResult.PerformData)) +} + +// allowedToUseMercury retrieves upkeep's administrative offchain config and decode a mercuryEnabled bool to indicate if +// this upkeep is allowed to use Mercury service. +func (s *streams) allowedToUseMercury(opts *bind.CallOpts, upkeepId *big.Int) (state mercury.MercuryUpkeepState, reason mercury.MercuryUpkeepFailureReason, retryable bool, allow bool, err error) { + allowed, ok := s.mercuryConfig.IsUpkeepAllowed(upkeepId.String()) + if ok { + return mercury.NoPipelineError, mercury.MercuryUpkeepFailureReasonNone, false, allowed.(bool), nil + } + + upkeepPrivilegeConfigBytes, err := s.registry.GetUpkeepPrivilegeConfig(opts, upkeepId) + if err != nil { + return mercury.PackUnpackDecodeFailed, mercury.MercuryUpkeepFailureReasonNone, false, false, fmt.Errorf("failed to get upkeep privilege config: %v", err) + } + + if len(upkeepPrivilegeConfigBytes) == 0 { + s.mercuryConfig.SetUpkeepAllowed(upkeepId.String(), false, cache.DefaultExpiration) + return mercury.NoPipelineError, mercury.MercuryUpkeepFailureReasonMercuryAccessNotAllowed, false, false, fmt.Errorf("upkeep privilege config is empty") + } + + var privilegeConfig UpkeepPrivilegeConfig + if err = json.Unmarshal(upkeepPrivilegeConfigBytes, &privilegeConfig); err != nil { + return mercury.MercuryUnmarshalError, mercury.MercuryUpkeepFailureReasonNone, false, false, fmt.Errorf("failed to unmarshal privilege config: %v", err) + } + + s.mercuryConfig.SetUpkeepAllowed(upkeepId.String(), privilegeConfig.MercuryEnabled, cache.DefaultExpiration) + + return mercury.NoPipelineError, mercury.MercuryUpkeepFailureReasonNone, false, privilegeConfig.MercuryEnabled, nil +} + +func (s *streams) checkCallback(ctx context.Context, values [][]byte, lookup *mercury.StreamsLookup) (mercury.MercuryUpkeepState, bool, iregistry21.CheckCallback, error) { + // call checkCallback function at the block which OCR3 has agreed upon + opts := s.buildCallOpts(ctx, new(big.Int).SetUint64(lookup.Block)) + checkCallback, err := s.registry.CheckCallback(opts, lookup.UpkeepId, values, lookup.ExtraData) + + if err != nil { + return mercury.RpcFlakyFailure, true, iregistry21.CheckCallback{}, err + } + + return mercury.NoPipelineError, false, checkCallback, nil +} + +func (s *streams) buildCallOpts(ctx context.Context, block *big.Int) *bind.CallOpts { + opts := bind.CallOpts{ + Context: ctx, + } + + if block == nil || block.Int64() == 0 { + if latestBlock := s.blockSubscriber.LatestBlock(); latestBlock != nil && latestBlock.Number != 0 { + opts.BlockNumber = big.NewInt(int64(latestBlock.Number)) + } + } else { + opts.BlockNumber = block + } + + return &opts +} + +// generatePluginRetryKey returns a plugin retry cache key +func generatePluginRetryKey(workID string, block uint64) string { + return workID + "|" + fmt.Sprintf("%d", block) +} diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/streams/streams_test.go b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/streams/streams_test.go new file mode 100644 index 00000000000..17c7e6682a7 --- /dev/null +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/streams/streams_test.go @@ -0,0 +1,719 @@ +package streams + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "math/big" + "net/http" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/pkg/errors" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + + "github.com/stretchr/testify/assert" + + //"github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" + "github.com/ethereum/go-ethereum/core/types" + + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated" + iregistry21 "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/i_keeper_registry_master_wrapper_2_1" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury" + v02 "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v02" + v03 "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v03" + + "github.com/ethereum/go-ethereum/common/hexutil" + ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + evmClientMocks "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client/mocks" + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/models" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/encoding" +) + +type MockMercuryConfigProvider struct { + mock.Mock +} + +func (m *MockMercuryConfigProvider) Credentials() *models.MercuryCredentials { + mc := &models.MercuryCredentials{ + LegacyURL: "https://google.old.com", + URL: "https://google.com", + Username: "FakeClientID", + Password: "FakeClientKey", + } + return mc +} + +func (m *MockMercuryConfigProvider) IsUpkeepAllowed(s string) (interface{}, bool) { + args := m.Called(s) + return args.Get(0), args.Bool(1) +} + +func (m *MockMercuryConfigProvider) SetUpkeepAllowed(s string, i interface{}, d time.Duration) { + m.Called(s, i, d) +} + +func (m *MockMercuryConfigProvider) GetPluginRetry(s string) (interface{}, bool) { + args := m.Called(s) + return args.Get(0), args.Bool(1) +} + +func (m *MockMercuryConfigProvider) SetPluginRetry(s string, i interface{}, d time.Duration) { + m.Called(s, i, d) +} + +type MockBlockSubscriber struct { + mock.Mock +} + +func (b *MockBlockSubscriber) LatestBlock() *ocr2keepers.BlockKey { + return nil +} + +type MockHttpClient struct { + mock.Mock +} + +func (mock *MockHttpClient) Do(req *http.Request) (*http.Response, error) { + args := mock.Called(req) + return args.Get(0).(*http.Response), args.Error(1) +} + +//type MockRegistry struct { +// mock.Mock +//} +// +//func (r *MockRegistry) GetUpkeepPrivilegeConfig(opts *bind.CallOpts, upkeepId *big.Int) ([]byte, error) { +// args := r.Called(opts, upkeepId) +// return args.Get(0).([]byte), args.Error(1) +//} +// +//func (r *MockRegistry) CheckCallback(opts *bind.CallOpts, id *big.Int, values [][]byte, extraData []byte) (iregistry21.CheckCallback, error) { +// args := r.Called(opts, id, values, extraData) +// return args.Get(0).(iregistry21.CheckCallback), args.Error(1) +//} + +//type IKeeperRegistryMasterCaller interface { +// // Define your methods if not already done in your code +//} +// +//type IKeeperRegistryMasterTransactor interface { +// // Define your methods if not already done in your code +//} +// +//type IKeeperRegistryMasterFilterer interface { +// // Define your methods if not already done in your code +//} +// +//type IKeeperRegistryMaster struct { +// address common.Address +// abi abi.ABI +// IKeeperRegistryMasterCaller +// IKeeperRegistryMasterTransactor +// IKeeperRegistryMasterFilterer +//} + +type MockIKeeperRegistryMaster struct { + mock.Mock + iregistry21.IKeeperRegistryMaster +} + +func (_IKeeperRegistryMaster *MockIKeeperRegistryMaster) Address() common.Address { + return common.HexToAddress("0x6cA639822c6C241Fa9A7A6b5032F6F7F1C513CAD") +} + +func (m *MockIKeeperRegistryMaster) ParseLog(log types.Log) (generated.AbigenLog, error) { + args := m.Called(log) + return args.Get(0).(generated.AbigenLog), args.Error(1) +} + +func (m *MockIKeeperRegistryMaster) GetUpkeepPrivilegeConfig(opts *bind.CallOpts, upkeepId *big.Int) ([]byte, error) { + args := m.Called(opts, upkeepId) + return args.Get(0).([]byte), args.Error(1) +} + +type mockRegistry struct { + GetUpkeepPrivilegeConfigFn func(opts *bind.CallOpts, upkeepId *big.Int) ([]byte, error) + CheckCallbackFn func(opts *bind.CallOpts, id *big.Int, values [][]byte, extraData []byte) (iregistry21.CheckCallback, error) +} + +func (r *mockRegistry) GetUpkeepPrivilegeConfig(opts *bind.CallOpts, upkeepId *big.Int) ([]byte, error) { + return r.GetUpkeepPrivilegeConfigFn(opts, upkeepId) +} + +func (r *mockRegistry) CheckCallback(opts *bind.CallOpts, id *big.Int, values [][]byte, extraData []byte) (iregistry21.CheckCallback, error) { + return r.CheckCallbackFn(opts, id, values, extraData) +} + +//// Your function using the registry +//func YourFunctionUsingRegistry(s *IKeeperRegistryMaster, opts *bind.CallOpts, upkeepId *big.Int) ([]byte, error) { +// upkeepPrivilegeConfigBytes, err := s.GetUpkeepPrivilegeConfig(opts, upkeepId) +// // Your remaining logic... +// return upkeepPrivilegeConfigBytes, err +//} + +// setups up a streams object for tests. +func setupStreams(t *testing.T) *streams { + lggr := logger.TestLogger(t) + //addr := common.HexToAddress("0x6cA639822c6C241Fa9A7A6b5032F6F7F1C513CAD") + packer := encoding.NewAbiPacker() + mercuryConfig := new(MockMercuryConfigProvider) + blockSubscriber := new(MockBlockSubscriber) + registry := &mockRegistry{ + GetUpkeepPrivilegeConfigFn: func(opts *bind.CallOpts, upkeepId *big.Int) ([]byte, error) { + return []byte{}, nil + }, + CheckCallbackFn: func(opts *bind.CallOpts, id *big.Int, values [][]byte, extraData []byte) (iregistry21.CheckCallback, error) { + return iregistry21.CheckCallback{}, nil + }, + } + //registry := &iregistry21.IKeeperRegistryMaster{} + + streams := NewStreamsLookup( + packer, + mercuryConfig, + blockSubscriber, + registry, + lggr, + ) + return streams +} + +func TestStreams_CheckCallback(t *testing.T) { + upkeepId := big.NewInt(123456789) + bn := uint64(999) + bs := []byte{183, 114, 215, 10, 0, 0, 0, 0, 0, 0} + values := [][]byte{bs} + tests := []struct { + name string + lookup *mercury.StreamsLookup + values [][]byte + statusCode int + + callbackResp []byte + callbackErr error + + upkeepNeeded bool + performData []byte + wantErr assert.ErrorAssertionFunc + + state mercury.MercuryUpkeepState + retryable bool + }{ + { + name: "success - empty extra data", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIdHex, + Feeds: []string{"ETD-USD", "BTC-ETH"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(100), + ExtraData: []byte{48, 120, 48, 48}, + }, + UpkeepId: upkeepId, + Block: bn, + }, + values: values, + statusCode: http.StatusOK, + callbackResp: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 48, 120, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + upkeepNeeded: true, + performData: []byte{48, 120, 48, 48}, + wantErr: assert.NoError, + }, + { + name: "success - with extra data", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIdHex, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(18952430), + // this is the address of precompile contract ArbSys(0x0000000000000000000000000000000000000064) + ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, + }, + UpkeepId: upkeepId, + Block: bn, + }, + values: values, + statusCode: http.StatusOK, + callbackResp: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + upkeepNeeded: true, + performData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, + wantErr: assert.NoError, + }, + { + name: "failure - bad response", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIdHex, + Feeds: []string{"ETD-USD", "BTC-ETH"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(100), + ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 48, 120, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + UpkeepId: upkeepId, + Block: bn, + }, + values: values, + statusCode: http.StatusOK, + callbackResp: []byte{}, + callbackErr: errors.New("bad response"), + wantErr: assert.Error, + state: mercury.RpcFlakyFailure, + retryable: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := new(evmClientMocks.Client) + r := setupStreams(t) + payload, err := r.abi.Pack("checkCallback", tt.lookup.UpkeepId, values, tt.lookup.ExtraData) + require.Nil(t, err) + args := map[string]interface{}{ + "to": r.addr.Hex(), + "data": hexutil.Bytes(payload), + } + client.On("CallContext", mock.Anything, mock.AnythingOfType("*hexutil.Bytes"), "eth_call", args, hexutil.EncodeUint64(tt.lookup.Block)).Return(tt.callbackErr). + Run(func(args mock.Arguments) { + by := args.Get(1).(*hexutil.Bytes) + *by = tt.callbackResp + }).Once() + r.client = client + + state, retryable, _, err := r.checkCallback(context.Background(), tt.values, tt.lookup) + tt.wantErr(t, err, fmt.Sprintf("Error asserion failed: %v", tt.name)) + assert.Equal(t, tt.state, state) + assert.Equal(t, tt.retryable, retryable) + }) + } +} + +func TestStreams_AllowedToUseMercury(t *testing.T) { + upkeepId, ok := new(big.Int).SetString("71022726777042968814359024671382968091267501884371696415772139504780367423725", 10) + assert.True(t, ok, t.Name()) + tests := []struct { + name string + cached bool + allowed bool + ethCallErr error + err error + state mercury.MercuryUpkeepState + reason mercury.MercuryUpkeepFailureReason + registry streamsRegistry + retryable bool + config []byte + }{ + { + name: "success - allowed via cache", + cached: true, + allowed: true, + registry: &mockRegistry{ + GetUpkeepPrivilegeConfigFn: func(opts *bind.CallOpts, upkeepId *big.Int) ([]byte, error) { + return nil, nil + }, + CheckCallbackFn: func(opts *bind.CallOpts, id *big.Int, values [][]byte, extraData []byte) (iregistry21.CheckCallback, error) { + return iregistry21.CheckCallback{}, nil + }, + }, + }, + { + name: "success - allowed via fetching privilege config", + allowed: true, + registry: &mockRegistry{ + GetUpkeepPrivilegeConfigFn: func(opts *bind.CallOpts, upkeepId *big.Int) ([]byte, error) { + return nil, nil + }, + CheckCallbackFn: func(opts *bind.CallOpts, id *big.Int, values [][]byte, extraData []byte) (iregistry21.CheckCallback, error) { + return iregistry21.CheckCallback{}, nil + }, + }, + }, + { + name: "success - not allowed via cache", + cached: true, + allowed: false, + registry: &mockRegistry{ + GetUpkeepPrivilegeConfigFn: func(opts *bind.CallOpts, upkeepId *big.Int) ([]byte, error) { + return nil, nil + }, + CheckCallbackFn: func(opts *bind.CallOpts, id *big.Int, values [][]byte, extraData []byte) (iregistry21.CheckCallback, error) { + return iregistry21.CheckCallback{}, nil + }, + }, + }, + { + name: "success - not allowed via fetching privilege config", + allowed: false, + registry: &mockRegistry{ + GetUpkeepPrivilegeConfigFn: func(opts *bind.CallOpts, upkeepId *big.Int) ([]byte, error) { + return nil, nil + }, + CheckCallbackFn: func(opts *bind.CallOpts, id *big.Int, values [][]byte, extraData []byte) (iregistry21.CheckCallback, error) { + return iregistry21.CheckCallback{}, nil + }, + }, + }, + { + name: "failure - cannot unmarshal privilege config", + err: fmt.Errorf("failed to unmarshal privilege config: invalid character '\\x00' looking for beginning of value"), + state: mercury.MercuryUnmarshalError, + config: []byte{0, 1}, + registry: &mockRegistry{ + GetUpkeepPrivilegeConfigFn: func(opts *bind.CallOpts, upkeepId *big.Int) ([]byte, error) { + return []byte(`{`), nil + }, + CheckCallbackFn: func(opts *bind.CallOpts, id *big.Int, values [][]byte, extraData []byte) (iregistry21.CheckCallback, error) { + return iregistry21.CheckCallback{}, nil + }, + }, + }, + { + name: "failure - cannot get privilege config", + err: fmt.Errorf("failed to unmarshal privilege config: invalid character '\\x00' looking for beginning of value"), + state: mercury.MercuryUnmarshalError, + config: []byte{0, 1}, + registry: &mockRegistry{ + GetUpkeepPrivilegeConfigFn: func(opts *bind.CallOpts, upkeepId *big.Int) ([]byte, error) { + return nil, errors.New("unable to get upkeep privilege config") + }, + CheckCallbackFn: func(opts *bind.CallOpts, id *big.Int, values [][]byte, extraData []byte) (iregistry21.CheckCallback, error) { + return iregistry21.CheckCallback{}, nil + }, + }, + }, + { + name: "failure - flaky RPC", + retryable: true, + err: fmt.Errorf("failed to get upkeep privilege config: flaky RPC"), + state: mercury.RpcFlakyFailure, + ethCallErr: fmt.Errorf("flaky RPC"), + }, + { + name: "failure - empty upkeep privilege config", + err: fmt.Errorf("upkeep privilege config is empty"), + reason: mercury.MercuryUpkeepFailureReasonMercuryAccessNotAllowed, + config: []byte{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := setupStreams(t) + s.registry = tt.registry + + client := new(evmClientMocks.Client) + s.client = client + + mc := new(MockMercuryConfigProvider) + mc.On("IsUpkeepAllowed", mock.Anything).Return(tt.allowed, tt.cached).Once() + mc.On("SetUpkeepAllowed", mock.Anything, mock.Anything, mock.Anything).Return().Once() + s.mercuryConfig = mc + + if !tt.cached { + if tt.err != nil { + bContractCfg, err := s.abi.Methods["getUpkeepPrivilegeConfig"].Outputs.PackValues([]interface{}{tt.config}) + require.Nil(t, err) + + payload, err := s.abi.Pack("getUpkeepPrivilegeConfig", upkeepId) + require.Nil(t, err) + + args := map[string]interface{}{ + "to": s.addr.Hex(), + "data": hexutil.Bytes(payload), + } + + client.On("CallContext", mock.Anything, mock.AnythingOfType("*hexutil.Bytes"), "eth_call", args, mock.AnythingOfType("string")). + Return(tt.ethCallErr). + Run(func(args mock.Arguments) { + b := args.Get(1).(*hexutil.Bytes) + *b = bContractCfg + }).Once() + } else { + cfg := UpkeepPrivilegeConfig{MercuryEnabled: tt.allowed} + bCfg, err := json.Marshal(cfg) + require.Nil(t, err) + + bContractCfg, err := s.abi.Methods["getUpkeepPrivilegeConfig"].Outputs.PackValues([]interface{}{bCfg}) + require.Nil(t, err) + + payload, err := s.abi.Pack("getUpkeepPrivilegeConfig", upkeepId) + require.Nil(t, err) + + args := map[string]interface{}{ + "to": s.addr.Hex(), + "data": hexutil.Bytes(payload), + } + + client.On("CallContext", mock.Anything, mock.AnythingOfType("*hexutil.Bytes"), "eth_call", args, mock.AnythingOfType("string")).Return(nil). + Run(func(args mock.Arguments) { + b := args.Get(1).(*hexutil.Bytes) + *b = bContractCfg + }).Once() + } + } + + opts := &bind.CallOpts{ + BlockNumber: big.NewInt(10), + } + + state, reason, retryable, allowed, err := s.allowedToUseMercury(opts, upkeepId) + assert.Equal(t, tt.err, err) + assert.Equal(t, tt.allowed, allowed) + assert.Equal(t, tt.state, state) + assert.Equal(t, tt.reason, reason) + assert.Equal(t, tt.retryable, retryable) + }) + } +} + +func TestStreams_StreamsLookup(t *testing.T) { + upkeepId, ok := new(big.Int).SetString("71022726777042968814359024671382968091267501884371696415772139504780367423725", 10) + var upkeepIdentifier [32]byte + copy(upkeepIdentifier[:], upkeepId.Bytes()) + assert.True(t, ok, t.Name()) + blockNum := ocr2keepers.BlockNumber(37974374) + tests := []struct { + name string + input []ocr2keepers.CheckResult + blobs map[string]string + callbackResp []byte + expectedResults []ocr2keepers.CheckResult + callbackNeeded bool + extraData []byte + checkCallbackResp []byte + values [][]byte + cachedAdminCfg bool + hasError bool + hasPermission bool + v3 bool + }{ + { + name: "success - happy path no cache", + input: []ocr2keepers.CheckResult{ + { + PerformData: hexutil.MustDecode("0xf055e4a200000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000e0000000000000000000000000000000000000000000000000000000000000024000000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000000000280000000000000000000000000000000000000000000000000000000000000000966656564496448657800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000423078343535343438326435353533343432643431353234323439353435323535346432643534343535333534346534353534303030303030303030303030303030300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000042307834323534343332643535353334343264343135323432343935343532353534643264353434353533353434653435353430303030303030303030303030303030000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000b626c6f636b4e756d62657200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000064000000000000000000000000"), + UpkeepID: upkeepIdentifier, + Trigger: ocr2keepers.Trigger{ + BlockNumber: blockNum, + }, + IneligibilityReason: uint8(mercury.MercuryUpkeepFailureReasonTargetCheckReverted), + }, + }, + blobs: map[string]string{ + "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000": "0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa3", + "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000": "0x0006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d", + }, + cachedAdminCfg: false, + extraData: hexutil.MustDecode("0x0000000000000000000000000000000000000064"), + callbackNeeded: true, + checkCallbackResp: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000063a400000000000000000000000000000000000000000000000000000000000006e0000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000006a000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000034000000000000000000000000000000000000000000000000000000000000002e000066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa300000000000000000000000000000000000000000000000000000000000002e00006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d00000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000064000000000000000000000000"), + values: [][]byte{hexutil.MustDecode("0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa3"), hexutil.MustDecode("0x0006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d")}, + expectedResults: []ocr2keepers.CheckResult{ + { + Eligible: true, + PerformData: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000006a000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000034000000000000000000000000000000000000000000000000000000000000002e000066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa300000000000000000000000000000000000000000000000000000000000002e00006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d00000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000064000000000000000000000000"), + UpkeepID: upkeepIdentifier, + Trigger: ocr2keepers.Trigger{ + BlockNumber: blockNum, + }, + IneligibilityReason: uint8(encoding.UpkeepFailureReasonNone), + }, + }, + hasPermission: true, + v3: false, + }, + { + name: "success - happy path no cache - v0.3", + input: []ocr2keepers.CheckResult{ + { + PerformData: hexutil.MustDecode("0xf055e4a200000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000e0000000000000000000000000000000000000000000000000000000000000024000000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000000000280000000000000000000000000000000000000000000000000000000000000000766656564494473000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000423078343535343438326435353533343432643431353234323439353435323535346432643534343535333534346534353534303030303030303030303030303030300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000042307834323534343332643535353334343264343135323432343935343532353534643264353434353533353434653435353430303030303030303030303030303030000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000b626c6f636b4e756d62657200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000064000000000000000000000000"), + UpkeepID: upkeepIdentifier, + Trigger: ocr2keepers.Trigger{ + BlockNumber: blockNum, + }, + IneligibilityReason: uint8(encoding.UpkeepFailureReasonTargetCheckReverted), + }, + }, + blobs: map[string]string{ + "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000": "0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa3", + "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000": "0x0006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d", + }, + cachedAdminCfg: false, + extraData: hexutil.MustDecode("0x0000000000000000000000000000000000000064"), + callbackNeeded: true, + checkCallbackResp: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000063a400000000000000000000000000000000000000000000000000000000000006e0000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000006a000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000034000000000000000000000000000000000000000000000000000000000000002e000066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa300000000000000000000000000000000000000000000000000000000000002e00006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d00000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000064000000000000000000000000"), + values: [][]byte{hexutil.MustDecode("0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa3"), hexutil.MustDecode("0x0006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d")}, + expectedResults: []ocr2keepers.CheckResult{ + { + Eligible: true, + PerformData: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000006a000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000034000000000000000000000000000000000000000000000000000000000000002e000066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa300000000000000000000000000000000000000000000000000000000000002e00006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d00000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000064000000000000000000000000"), + UpkeepID: upkeepIdentifier, + Trigger: ocr2keepers.Trigger{ + BlockNumber: blockNum, + }, + IneligibilityReason: uint8(encoding.UpkeepFailureReasonNone), + }, + }, + hasPermission: true, + v3: true, + }, + { + name: "skip - failure reason is insufficient balance", + input: []ocr2keepers.CheckResult{ + { + PerformData: []byte{}, + UpkeepID: upkeepIdentifier, + Trigger: ocr2keepers.Trigger{ + BlockNumber: 26046145, + }, + IneligibilityReason: uint8(encoding.UpkeepFailureReasonInsufficientBalance), + }, + }, + expectedResults: []ocr2keepers.CheckResult{ + { + Eligible: false, + PerformData: []byte{}, + UpkeepID: upkeepIdentifier, + Trigger: ocr2keepers.Trigger{ + BlockNumber: 26046145, + }, + IneligibilityReason: uint8(encoding.UpkeepFailureReasonInsufficientBalance), + }, + }, + hasError: true, + }, + { + name: "skip - invalid revert data", + input: []ocr2keepers.CheckResult{ + { + PerformData: []byte{}, + UpkeepID: upkeepIdentifier, + Trigger: ocr2keepers.Trigger{ + BlockNumber: 26046145, + }, + IneligibilityReason: uint8(encoding.UpkeepFailureReasonTargetCheckReverted), + }, + }, + expectedResults: []ocr2keepers.CheckResult{ + { + Eligible: false, + PerformData: []byte{}, + UpkeepID: upkeepIdentifier, + Trigger: ocr2keepers.Trigger{ + BlockNumber: 26046145, + }, + IneligibilityReason: uint8(encoding.UpkeepFailureReasonTargetCheckReverted), + }, + }, + hasError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := setupStreams(t) + client := new(evmClientMocks.Client) + s.client = client + + mc := new(MockMercuryConfigProvider) + mc.On("IsUpkeepAllowed", mock.Anything).Return(tt.hasPermission, tt.cachedAdminCfg).Once() + mc.On("SetUpkeepAllowed", mock.Anything, mock.Anything, mock.Anything).Return().Once() + s.mercuryConfig = mc + + if !tt.cachedAdminCfg && !tt.hasError { + cfg := UpkeepPrivilegeConfig{MercuryEnabled: tt.hasPermission} + bCfg, err := json.Marshal(cfg) + require.Nil(t, err) + + bContractCfg, err := s.abi.Methods["getUpkeepPrivilegeConfig"].Outputs.PackValues([]interface{}{bCfg}) + require.Nil(t, err) + + payload, err := s.abi.Pack("getUpkeepPrivilegeConfig", upkeepId) + require.Nil(t, err) + + args := map[string]interface{}{ + "to": s.addr.Hex(), + "data": hexutil.Bytes(payload), + } + + client.On("CallContext", mock.Anything, mock.AnythingOfType("*hexutil.Bytes"), "eth_call", args, mock.AnythingOfType("string")).Return(nil). + Run(func(args mock.Arguments) { + b := args.Get(1).(*hexutil.Bytes) + *b = bContractCfg + }).Once() + } + + if len(tt.blobs) > 0 { + if tt.v3 { + hc03 := new(MockHttpClient) + v03HttpClient := v03.NewClient(s.mercuryConfig, hc03, s.lggr) + s.v03Client = v03HttpClient + + mr1 := v03.MercuryV03Response{ + Reports: []v03.MercuryV03Report{{FullReport: tt.blobs["0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"]}, {FullReport: tt.blobs["0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"]}}} + b1, err := json.Marshal(mr1) + assert.Nil(t, err) + resp1 := &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader(b1)), + } + hc03.On("Do", mock.Anything).Return(resp1, nil).Once() + } else { + hc02 := new(MockHttpClient) + v02HttpClient := v02.NewClient(s.mercuryConfig, hc02, s.lggr) + s.v02Client = v02HttpClient + + mr1 := v02.MercuryV02Response{ChainlinkBlob: tt.blobs["0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"]} + b1, err := json.Marshal(mr1) + assert.Nil(t, err) + resp1 := &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader(b1)), + } + mr2 := v02.MercuryV02Response{ChainlinkBlob: tt.blobs["0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"]} + b2, err := json.Marshal(mr2) + assert.Nil(t, err) + resp2 := &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader(b2)), + } + + hc02.On("Do", mock.MatchedBy(func(req *http.Request) bool { + return strings.Contains(req.URL.String(), "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000") + })).Return(resp2, nil).Once() + + hc02.On("Do", mock.MatchedBy(func(req *http.Request) bool { + return strings.Contains(req.URL.String(), "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000") + })).Return(resp1, nil).Once() + } + } + + if tt.callbackNeeded { + payload, err := s.abi.Pack("checkCallback", upkeepId, tt.values, tt.extraData) + require.Nil(t, err) + args := map[string]interface{}{ + "to": s.addr.Hex(), + "data": hexutil.Bytes(payload), + } + client.On("CallContext", mock.Anything, mock.AnythingOfType("*hexutil.Bytes"), "eth_call", args, hexutil.EncodeUint64(uint64(blockNum))).Return(nil). + Run(func(args mock.Arguments) { + b := args.Get(1).(*hexutil.Bytes) + *b = tt.checkCallbackResp + }).Once() + } + + got := s.Lookup(context.Background(), tt.input) + assert.Equal(t, tt.expectedResults, got, tt.name) + }) + } +} diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/upkeep_failure_reasons.go b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/upkeep_failure_reasons.go new file mode 100644 index 00000000000..261fc33bd46 --- /dev/null +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/upkeep_failure_reasons.go @@ -0,0 +1,15 @@ +package mercury + +type MercuryUpkeepFailureReason uint8 + +const ( + // upkeep failure onchain reasons + MercuryUpkeepFailureReasonNone MercuryUpkeepFailureReason = 0 + MercuryUpkeepFailureReasonTargetCheckReverted MercuryUpkeepFailureReason = 3 + MercuryUpkeepFailureReasonUpkeepNotNeeded MercuryUpkeepFailureReason = 4 + MercuryUpkeepFailureReasonMercuryCallbackReverted MercuryUpkeepFailureReason = 7 + // leaving a gap here for more onchain failure reasons in the future + // upkeep failure offchain reasons + MercuryUpkeepFailureReasonMercuryAccessNotAllowed MercuryUpkeepFailureReason = 32 + MercuryUpkeepFailureReasonInvalidRevertDataInput MercuryUpkeepFailureReason = 34 +) diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/upkeep_states.go b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/upkeep_states.go new file mode 100644 index 00000000000..4e9cd14aee3 --- /dev/null +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/upkeep_states.go @@ -0,0 +1,14 @@ +package mercury + +type MercuryUpkeepState uint8 + +const ( + NoPipelineError MercuryUpkeepState = 0 + RpcFlakyFailure MercuryUpkeepState = 3 + MercuryFlakyFailure MercuryUpkeepState = 4 + PackUnpackDecodeFailed MercuryUpkeepState = 5 + MercuryUnmarshalError MercuryUpkeepState = 6 + InvalidMercuryRequest MercuryUpkeepState = 7 + InvalidMercuryResponse MercuryUpkeepState = 8 // this will only happen if Mercury server sends bad responses + UpkeepNotAuthorized MercuryUpkeepState = 9 +) diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v02/request.go b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v02/request.go new file mode 100644 index 00000000000..ad88c0a68bb --- /dev/null +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v02/request.go @@ -0,0 +1,188 @@ +package v02 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/avast/retry-go/v4" + "github.com/ethereum/go-ethereum/common/hexutil" + + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury" +) + +const ( + mercuryPathV02 = "/client?" // only used to access mercury v0.2 server + retryDelay = 500 * time.Millisecond + totalAttempt = 3 + contentTypeHeader = "Content-Type" + authorizationHeader = "Authorization" + timestampHeader = "X-Authorization-Timestamp" + signatureHeader = "X-Authorization-Signature-SHA256" +) + +type MercuryV02Response struct { + ChainlinkBlob string `json:"chainlinkBlob"` +} + +type client struct { + mercuryConfig mercury.MercuryConfigProvider + httpClient mercury.HttpClient + lggr logger.Logger +} + +func NewClient(mercuryConfig mercury.MercuryConfigProvider, httpClient mercury.HttpClient, lggr logger.Logger) *client { + return &client{ + mercuryConfig: mercuryConfig, + httpClient: httpClient, + lggr: lggr, + } +} + +func (c *client) DoRequest(ctx context.Context, streamsLookup *mercury.StreamsLookup, pluginRetryKey string) (mercury.MercuryUpkeepState, mercury.MercuryUpkeepFailureReason, [][]byte, bool, time.Duration, error) { + resultLen := len(streamsLookup.Feeds) + ch := make(chan mercury.MercuryData, resultLen) + if len(streamsLookup.Feeds) == 0 { + return mercury.NoPipelineError, mercury.MercuryUpkeepFailureReasonInvalidRevertDataInput, [][]byte{}, false, 0 * time.Second, fmt.Errorf("invalid revert data input: feed param key %s, time param key %s, feeds %s", streamsLookup.FeedParamKey, streamsLookup.TimeParamKey, streamsLookup.Feeds) + } + for i := range streamsLookup.Feeds { + go c.singleFeedRequest(ctx, ch, i, streamsLookup) + } + + var reqErr error + var retryInterval time.Duration + results := make([][]byte, len(streamsLookup.Feeds)) + retryable := true + allSuccess := true + // in v0.2, use the last execution error as the state, if no execution errors, state will be no error + state := mercury.NoPipelineError + for i := 0; i < resultLen; i++ { + m := <-ch + if m.Error != nil { + reqErr = errors.Join(reqErr, m.Error) + retryable = retryable && m.Retryable + allSuccess = false + if m.State != mercury.NoPipelineError { + state = mercury.MercuryUpkeepState(m.State) + } + continue + } + results[m.Index] = m.Bytes[0] + } + if retryable && !allSuccess { + retryInterval = mercury.CalculateRetryConfigFn(pluginRetryKey, c.mercuryConfig) + } + // only retry when not all successful AND none are not retryable + return state, mercury.MercuryUpkeepFailureReasonNone, results, retryable && !allSuccess, retryInterval, reqErr +} + +func (c *client) singleFeedRequest(ctx context.Context, ch chan<- mercury.MercuryData, index int, sl *mercury.StreamsLookup) { + var httpRequest *http.Request + var err error + + q := url.Values{ + sl.FeedParamKey: {sl.Feeds[index]}, + sl.TimeParamKey: {sl.Time.String()}, + } + mercuryURL := c.mercuryConfig.Credentials().LegacyURL + reqUrl := fmt.Sprintf("%s%s%s", mercuryURL, mercuryPathV02, q.Encode()) + c.lggr.Debugf("request URL for upkeep %s feed %s: %s", sl.UpkeepId.String(), sl.Feeds[index], reqUrl) + + httpRequest, err = http.NewRequestWithContext(ctx, http.MethodGet, reqUrl, nil) + if err != nil { + ch <- mercury.MercuryData{Index: index, Error: err, Retryable: false, State: mercury.InvalidMercuryRequest} + return + } + + ts := time.Now().UTC().UnixMilli() + signature := mercury.GenerateHMACFn(http.MethodGet, mercuryPathV02+q.Encode(), []byte{}, c.mercuryConfig.Credentials().Username, c.mercuryConfig.Credentials().Password, ts) + httpRequest.Header.Set(contentTypeHeader, "application/json") + httpRequest.Header.Set(authorizationHeader, c.mercuryConfig.Credentials().Username) + httpRequest.Header.Set(timestampHeader, strconv.FormatInt(ts, 10)) + httpRequest.Header.Set(signatureHeader, signature) + + // in the case of multiple retries here, use the last attempt's data + state := mercury.NoPipelineError + retryable := false + sent := false + retryErr := retry.Do( + func() error { + var httpResponse *http.Response + var responseBody []byte + var blobBytes []byte + + retryable = false + if httpResponse, err = c.httpClient.Do(httpRequest); err != nil { + c.lggr.Warnf("at block %s upkeep %s GET request fails for feed %s: %v", sl.Time.String(), sl.UpkeepId.String(), sl.Feeds[index], err) + retryable = true + state = mercury.MercuryFlakyFailure + return err + } + defer httpResponse.Body.Close() + + if responseBody, err = io.ReadAll(httpResponse.Body); err != nil { + state = mercury.InvalidMercuryResponse + return err + } + + switch httpResponse.StatusCode { + case http.StatusNotFound, http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout: + c.lggr.Warnf("at block %s upkeep %s received status code %d for feed %s", sl.Time.String(), sl.UpkeepId.String(), httpResponse.StatusCode, sl.Feeds[index]) + retryable = true + state = mercury.MercuryFlakyFailure + return errors.New(strconv.FormatInt(int64(httpResponse.StatusCode), 10)) + case http.StatusOK: + // continue + default: + state = mercury.InvalidMercuryRequest + return fmt.Errorf("at block %s upkeep %s received status code %d for feed %s", sl.Time.String(), sl.UpkeepId.String(), httpResponse.StatusCode, sl.Feeds[index]) + } + + c.lggr.Debugf("at block %s upkeep %s received status code %d from mercury v0.2 with BODY=%s", sl.Time.String(), sl.UpkeepId.String(), httpResponse.StatusCode, hexutil.Encode(responseBody)) + + var m MercuryV02Response + if err = json.Unmarshal(responseBody, &m); err != nil { + c.lggr.Warnf("at block %s upkeep %s failed to unmarshal body to MercuryV02Response for feed %s: %v", sl.Time.String(), sl.UpkeepId.String(), sl.Feeds[index], err) + state = mercury.MercuryUnmarshalError + return err + } + if blobBytes, err = hexutil.Decode(m.ChainlinkBlob); err != nil { + c.lggr.Warnf("at block %s upkeep %s failed to decode chainlinkBlob %s for feed %s: %v", sl.Time.String(), sl.UpkeepId.String(), m.ChainlinkBlob, sl.Feeds[index], err) + state = mercury.InvalidMercuryResponse + return err + } + ch <- mercury.MercuryData{ + Index: index, + Bytes: [][]byte{blobBytes}, + Retryable: false, + State: mercury.NoPipelineError, + } + sent = true + return nil + }, + // only retry when the error is 404 Not Found, 500 Internal Server Error, 502 Bad Gateway, 503 Service Unavailable, 504 Gateway Timeout + retry.RetryIf(func(err error) bool { + return err.Error() == fmt.Sprintf("%d", http.StatusNotFound) || err.Error() == fmt.Sprintf("%d", http.StatusInternalServerError) || err.Error() == fmt.Sprintf("%d", http.StatusBadGateway) || err.Error() == fmt.Sprintf("%d", http.StatusServiceUnavailable) || err.Error() == fmt.Sprintf("%d", http.StatusGatewayTimeout) + }), + retry.Context(ctx), + retry.Delay(retryDelay), + retry.Attempts(totalAttempt), + ) + + if !sent { + ch <- mercury.MercuryData{ + Index: index, + Bytes: [][]byte{}, + Retryable: retryable, + Error: fmt.Errorf("failed to request feed for %s: %w", sl.Feeds[index], retryErr), + State: state, + } + } +} diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v02/v02_request_test.go b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v02/v02_request_test.go new file mode 100644 index 00000000000..87e4ffc6725 --- /dev/null +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v02/v02_request_test.go @@ -0,0 +1,463 @@ +package v02 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "math/big" + "net/http" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/patrickmn/go-cache" + "github.com/stretchr/testify/mock" + + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/models" + + "github.com/stretchr/testify/assert" + + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury" +) + +const ( + defaultPluginRetryExpiration = 30 * time.Minute + cleanupInterval = 5 * time.Minute +) + +type MockMercuryConfigProvider struct { + cache *cache.Cache + mock.Mock +} + +func NewMockMercuryConfigProvider() *MockMercuryConfigProvider { + return &MockMercuryConfigProvider{ + cache: cache.New(defaultPluginRetryExpiration, cleanupInterval), + } +} + +func (m *MockMercuryConfigProvider) Credentials() *models.MercuryCredentials { + mc := &models.MercuryCredentials{ + LegacyURL: "https://google.old.com", + URL: "https://google.com", + Username: "FakeClientID", + Password: "FakeClientKey", + } + return mc +} + +func (m *MockMercuryConfigProvider) IsUpkeepAllowed(s string) (interface{}, bool) { + args := m.Called(s) + return args.Get(0), args.Bool(1) +} + +func (m *MockMercuryConfigProvider) SetUpkeepAllowed(s string, i interface{}, d time.Duration) { + m.Called(s, i, d) +} + +func (m *MockMercuryConfigProvider) GetPluginRetry(s string) (interface{}, bool) { + if value, found := m.cache.Get(s); found { + return value, true + } + + return nil, false +} + +func (m *MockMercuryConfigProvider) SetPluginRetry(s string, i interface{}, d time.Duration) { + m.cache.Set(s, i, d) +} + +type MockHttpClient struct { + mock.Mock +} + +func (mock *MockHttpClient) Do(req *http.Request) (*http.Response, error) { + args := mock.Called(req) + return args.Get(0).(*http.Response), args.Error(1) +} + +// setups up a client object for tests. +func setupClient(t *testing.T) *client { + lggr := logger.TestLogger(t) + mockHttpClient := new(MockHttpClient) + mercuryConfig := NewMockMercuryConfigProvider() + + client := NewClient( + mercuryConfig, + mockHttpClient, + lggr, + ) + return client +} + +func TestV02_SingleFeedRequest(t *testing.T) { + upkeepId := big.NewInt(123456789) + tests := []struct { + name string + index int + lookup *mercury.StreamsLookup + blob string + statusCode int + lastStatusCode int + retryNumber int + retryable bool + errorMessage string + }{ + { + name: "success - mercury responds in the first try", + index: 0, + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIdHex, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + blob: "0xab2123dc00000012", + }, + { + name: "success - retry for 404", + index: 0, + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIdHex, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + blob: "0xab2123dcbabbad", + retryNumber: 1, + statusCode: http.StatusNotFound, + lastStatusCode: http.StatusOK, + }, + { + name: "success - retry for 500", + index: 0, + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIdHex, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + blob: "0xab2123dcbbabad", + retryNumber: 2, + statusCode: http.StatusInternalServerError, + lastStatusCode: http.StatusOK, + }, + { + name: "failure - returns retryable", + index: 0, + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIdHex, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + blob: "0xab2123dc", + retryNumber: totalAttempt, + statusCode: http.StatusNotFound, + retryable: true, + errorMessage: "failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000: All attempts fail:\n#1: 404\n#2: 404\n#3: 404", + }, + { + name: "failure - returns retryable and then non-retryable", + index: 0, + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIdHex, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + blob: "0xab2123dc", + retryNumber: 1, + statusCode: http.StatusNotFound, + lastStatusCode: http.StatusTooManyRequests, + errorMessage: "failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000: All attempts fail:\n#1: 404\n#2: at block 123456 upkeep 123456789 received status code 429 for feed 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", + }, + { + name: "failure - returns not retryable", + index: 0, + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIdHex, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + blob: "0xab2123dc", + statusCode: http.StatusConflict, + errorMessage: "failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000: All attempts fail:\n#1: at block 123456 upkeep 123456789 received status code 409 for feed 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := setupClient(t) + hc := new(MockHttpClient) + + mr := MercuryV02Response{ChainlinkBlob: tt.blob} + b, err := json.Marshal(mr) + assert.Nil(t, err) + + if tt.retryNumber == 0 { + if tt.errorMessage != "" { + resp := &http.Response{ + StatusCode: tt.statusCode, + Body: io.NopCloser(bytes.NewReader(b)), + } + hc.On("Do", mock.Anything).Return(resp, nil).Once() + } else { + resp := &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader(b)), + } + hc.On("Do", mock.Anything).Return(resp, nil).Once() + } + } else if tt.retryNumber > 0 && tt.retryNumber < totalAttempt { + retryResp := &http.Response{ + StatusCode: tt.statusCode, + Body: io.NopCloser(bytes.NewReader(b)), + } + hc.On("Do", mock.Anything).Return(retryResp, nil).Times(tt.retryNumber) + + resp := &http.Response{ + StatusCode: tt.lastStatusCode, + Body: io.NopCloser(bytes.NewReader(b)), + } + hc.On("Do", mock.Anything).Return(resp, nil).Once() + } else { + resp := &http.Response{ + StatusCode: tt.statusCode, + Body: io.NopCloser(bytes.NewReader(b)), + } + hc.On("Do", mock.Anything).Return(resp, nil).Times(tt.retryNumber) + } + c.httpClient = hc + + ch := make(chan mercury.MercuryData, 1) + c.singleFeedRequest(context.Background(), ch, tt.index, tt.lookup) + + m := <-ch + assert.Equal(t, tt.index, m.Index) + assert.Equal(t, tt.retryable, m.Retryable) + if tt.retryNumber >= totalAttempt || tt.errorMessage != "" { + assert.Equal(t, tt.errorMessage, m.Error.Error()) + assert.Equal(t, [][]byte{}, m.Bytes) + } else { + blobBytes, err := hexutil.Decode(tt.blob) + assert.Nil(t, err) + assert.Nil(t, m.Error) + assert.Equal(t, [][]byte{blobBytes}, m.Bytes) + } + }) + } +} + +func TestV02_DoMercuryRequestV02(t *testing.T) { + upkeepId, _ := new(big.Int).SetString("88786950015966611018675766524283132478093844178961698330929478019253453382042", 10) + + tests := []struct { + name string + lookup *mercury.StreamsLookup + mockHttpStatusCode int + mockChainlinkBlobs []string + pluginRetries int + pluginRetryKey string + expectedValues [][]byte + expectedRetryable bool + expectedRetryInterval time.Duration + expectedError error + state mercury.MercuryUpkeepState + reason mercury.MercuryUpkeepFailureReason + }{ + { + name: "success", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIdHex, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(25880526), + ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, + }, + UpkeepId: upkeepId, + }, + mockHttpStatusCode: http.StatusOK, + mockChainlinkBlobs: []string{"0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000000081401000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000289ad8d367000000000000000000000000000000000000000000000000000000289acf0b38000000000000000000000000000000000000000000000000000000289b3da40000000000000000000000000000000000000000000000000000000000018ae7ce74d9fa252a8983976eab600dc7590c778d04813430841bc6e765c34cd81a168d00000000000000000000000000000000000000000000000000000000018ae7cb0000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000000000000260412b94e525ca6cedc9f544fd86f77606d52fe731a5d069dbe836a8bfc0fb8c911963b0ae7a14971f3b4621bffb802ef0605392b9a6c89c7fab1df8633a5ade00000000000000000000000000000000000000000000000000000000000000024500c2f521f83fba5efc2bf3effaaedde43d0a4adff785c1213b712a3aed0d8157642a84324db0cf9695ebd27708d4608eb0337e0dd87b0e43f0fa70c700d911"}, + expectedValues: [][]byte{{0, 6, 109, 252, 209, 237, 45, 149, 177, 140, 148, 141, 188, 91, 214, 76, 104, 122, 254, 147, 228, 202, 125, 102, 61, 222, 193, 76, 32, 9, 10, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 128, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 69, 84, 72, 45, 85, 83, 68, 45, 65, 82, 66, 73, 84, 82, 85, 77, 45, 84, 69, 83, 84, 78, 69, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 137, 28, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 154, 216, 211, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 154, 207, 11, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 155, 61, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 138, 231, 206, 116, 217, 250, 37, 42, 137, 131, 151, 110, 171, 96, 13, 199, 89, 12, 119, 141, 4, 129, 52, 48, 132, 27, 198, 231, 101, 195, 76, 216, 26, 22, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 138, 231, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 137, 28, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 96, 65, 43, 148, 229, 37, 202, 108, 237, 201, 245, 68, 253, 134, 247, 118, 6, 213, 47, 231, 49, 165, 208, 105, 219, 232, 54, 168, 191, 192, 251, 140, 145, 25, 99, 176, 174, 122, 20, 151, 31, 59, 70, 33, 191, 251, 128, 46, 240, 96, 83, 146, 185, 166, 200, 156, 127, 171, 29, 248, 99, 58, 90, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 69, 0, 194, 245, 33, 248, 63, 186, 94, 252, 43, 243, 239, 250, 174, 221, 228, 61, 10, 74, 223, 247, 133, 193, 33, 59, 113, 42, 58, 237, 13, 129, 87, 100, 42, 132, 50, 77, 176, 207, 150, 149, 235, 210, 119, 8, 212, 96, 142, 176, 51, 126, 13, 216, 123, 14, 67, 240, 250, 112, 199, 0, 217, 17}}, + expectedRetryable: false, + expectedError: nil, + }, + { + name: "failure - retryable and interval is 1s", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIdHex, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(25880526), + ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, + }, + UpkeepId: upkeepId, + }, + mockHttpStatusCode: http.StatusInternalServerError, + mockChainlinkBlobs: []string{"0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000000081401000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000289ad8d367000000000000000000000000000000000000000000000000000000289acf0b38000000000000000000000000000000000000000000000000000000289b3da40000000000000000000000000000000000000000000000000000000000018ae7ce74d9fa252a8983976eab600dc7590c778d04813430841bc6e765c34cd81a168d00000000000000000000000000000000000000000000000000000000018ae7cb0000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000000000000260412b94e525ca6cedc9f544fd86f77606d52fe731a5d069dbe836a8bfc0fb8c911963b0ae7a14971f3b4621bffb802ef0605392b9a6c89c7fab1df8633a5ade00000000000000000000000000000000000000000000000000000000000000024500c2f521f83fba5efc2bf3effaaedde43d0a4adff785c1213b712a3aed0d8157642a84324db0cf9695ebd27708d4608eb0337e0dd87b0e43f0fa70c700d911"}, + expectedValues: [][]byte{nil}, + expectedRetryable: true, + pluginRetries: 0, + expectedRetryInterval: 1 * time.Second, + expectedError: errors.New("failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000: All attempts fail:\n#1: 500\n#2: 500\n#3: 500"), + state: mercury.MercuryFlakyFailure, + }, + { + name: "failure - retryable and interval is 5s", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIdHex, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(25880526), + ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, + }, + UpkeepId: upkeepId, + }, + pluginRetries: 5, + mockHttpStatusCode: http.StatusInternalServerError, + mockChainlinkBlobs: []string{"0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000000081401000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000289ad8d367000000000000000000000000000000000000000000000000000000289acf0b38000000000000000000000000000000000000000000000000000000289b3da40000000000000000000000000000000000000000000000000000000000018ae7ce74d9fa252a8983976eab600dc7590c778d04813430841bc6e765c34cd81a168d00000000000000000000000000000000000000000000000000000000018ae7cb0000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000000000000260412b94e525ca6cedc9f544fd86f77606d52fe731a5d069dbe836a8bfc0fb8c911963b0ae7a14971f3b4621bffb802ef0605392b9a6c89c7fab1df8633a5ade00000000000000000000000000000000000000000000000000000000000000024500c2f521f83fba5efc2bf3effaaedde43d0a4adff785c1213b712a3aed0d8157642a84324db0cf9695ebd27708d4608eb0337e0dd87b0e43f0fa70c700d911"}, + expectedValues: [][]byte{nil}, + expectedRetryable: true, + expectedRetryInterval: 5 * time.Second, + expectedError: errors.New("failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000: All attempts fail:\n#1: 500\n#2: 500\n#3: 500"), + state: mercury.MercuryFlakyFailure, + }, + { + name: "failure - not retryable because there are many plugin retries already", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIdHex, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(25880526), + ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, + }, + UpkeepId: upkeepId, + }, + pluginRetries: 10, + mockHttpStatusCode: http.StatusInternalServerError, + mockChainlinkBlobs: []string{"0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000000081401000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000289ad8d367000000000000000000000000000000000000000000000000000000289acf0b38000000000000000000000000000000000000000000000000000000289b3da40000000000000000000000000000000000000000000000000000000000018ae7ce74d9fa252a8983976eab600dc7590c778d04813430841bc6e765c34cd81a168d00000000000000000000000000000000000000000000000000000000018ae7cb0000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000000000000260412b94e525ca6cedc9f544fd86f77606d52fe731a5d069dbe836a8bfc0fb8c911963b0ae7a14971f3b4621bffb802ef0605392b9a6c89c7fab1df8633a5ade00000000000000000000000000000000000000000000000000000000000000024500c2f521f83fba5efc2bf3effaaedde43d0a4adff785c1213b712a3aed0d8157642a84324db0cf9695ebd27708d4608eb0337e0dd87b0e43f0fa70c700d911"}, + expectedValues: [][]byte{nil}, + expectedRetryable: true, + expectedError: errors.New("failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000: All attempts fail:\n#1: 500\n#2: 500\n#3: 500"), + state: mercury.MercuryFlakyFailure, + }, + { + name: "failure - not retryable", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIdHex, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(25880526), + ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, + }, + UpkeepId: upkeepId, + }, + mockHttpStatusCode: http.StatusTooManyRequests, + mockChainlinkBlobs: []string{"0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000000081401000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000289ad8d367000000000000000000000000000000000000000000000000000000289acf0b38000000000000000000000000000000000000000000000000000000289b3da40000000000000000000000000000000000000000000000000000000000018ae7ce74d9fa252a8983976eab600dc7590c778d04813430841bc6e765c34cd81a168d00000000000000000000000000000000000000000000000000000000018ae7cb0000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000000000000260412b94e525ca6cedc9f544fd86f77606d52fe731a5d069dbe836a8bfc0fb8c911963b0ae7a14971f3b4621bffb802ef0605392b9a6c89c7fab1df8633a5ade00000000000000000000000000000000000000000000000000000000000000024500c2f521f83fba5efc2bf3effaaedde43d0a4adff785c1213b712a3aed0d8157642a84324db0cf9695ebd27708d4608eb0337e0dd87b0e43f0fa70c700d911"}, + expectedValues: [][]byte{nil}, + expectedRetryable: false, + expectedError: errors.New("failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000: All attempts fail:\n#1: at block 25880526 upkeep 88786950015966611018675766524283132478093844178961698330929478019253453382042 received status code 429 for feed 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"), + state: mercury.InvalidMercuryRequest, + }, + { + name: "failure - no feeds", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIdHex, + Feeds: []string{}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(25880526), + ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, + }, + UpkeepId: upkeepId, + }, + expectedValues: [][]byte{}, + reason: mercury.MercuryUpkeepFailureReasonInvalidRevertDataInput, + }, + { + name: "failure - invalid revert data", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIDs, + Feeds: []string{}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(25880526), + ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, + }, + UpkeepId: upkeepId, + }, + expectedValues: [][]byte{}, + reason: mercury.MercuryUpkeepFailureReasonInvalidRevertDataInput, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := setupClient(t) + if tt.pluginRetries != 0 { + c.mercuryConfig.SetPluginRetry(tt.pluginRetryKey, tt.pluginRetries, cache.DefaultExpiration) + } + hc := new(MockHttpClient) + + for _, blob := range tt.mockChainlinkBlobs { + mr := MercuryV02Response{ChainlinkBlob: blob} + b, err := json.Marshal(mr) + assert.Nil(t, err) + + resp := &http.Response{ + StatusCode: tt.mockHttpStatusCode, + Body: io.NopCloser(bytes.NewReader(b)), + } + if tt.expectedError != nil && tt.expectedRetryable || tt.pluginRetries > 0 { + hc.On("Do", mock.Anything).Return(resp, nil).Times(totalAttempt) + } else { + hc.On("Do", mock.Anything).Return(resp, nil).Once() + } + } + c.httpClient = hc + + state, reason, values, retryable, retryInterval, reqErr := c.DoRequest(context.Background(), tt.lookup, tt.pluginRetryKey) + assert.Equal(t, tt.expectedValues, values) + assert.Equal(t, tt.expectedRetryable, retryable) + if retryable { + newRetries, _ := c.mercuryConfig.GetPluginRetry(tt.pluginRetryKey) + assert.Equal(t, tt.pluginRetries+1, newRetries.(int)) + } + assert.Equal(t, tt.expectedRetryInterval, retryInterval) + assert.Equal(t, tt.state, state) + assert.Equal(t, tt.reason, reason) + if tt.expectedError != nil { + assert.True(t, strings.HasPrefix(reqErr.Error(), "failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000")) + } + }) + } +} diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v03/request.go b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v03/request.go new file mode 100644 index 00000000000..0f46277218f --- /dev/null +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v03/request.go @@ -0,0 +1,239 @@ +package v03 + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/avast/retry-go/v4" + "github.com/ethereum/go-ethereum/common/hexutil" + + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury" +) + +const ( + mercuryBatchPathV03 = "/api/v1/reports/bulk?" // only used to access mercury v0.3 server + mercuryBatchPathV03BlockNumber = "/api/v1gmx/reports/bulk?" // only used to access mercury v0.3 server with blockNumber + retryDelay = 500 * time.Millisecond + totalAttempt = 3 + contentTypeHeader = "Content-Type" + authorizationHeader = "Authorization" + timestampHeader = "X-Authorization-Timestamp" + signatureHeader = "X-Authorization-Signature-SHA256" + upkeepIDHeader = "X-Authorization-Upkeep-Id" +) + +type MercuryV03Response struct { + Reports []MercuryV03Report `json:"reports"` +} + +type MercuryV03Report struct { + FeedID string `json:"feedID"` // feed id in hex encoded + ValidFromTimestamp uint32 `json:"validFromTimestamp"` + ObservationsTimestamp uint32 `json:"observationsTimestamp"` + FullReport string `json:"fullReport"` // the actual hex encoded mercury report of this feed, can be sent to verifier +} + +type client struct { + mercuryConfig mercury.MercuryConfigProvider + httpClient mercury.HttpClient + lggr logger.Logger +} + +func NewClient(mercuryConfig mercury.MercuryConfigProvider, httpClient mercury.HttpClient, lggr logger.Logger) *client { + return &client{ + mercuryConfig: mercuryConfig, + httpClient: httpClient, + lggr: lggr, + } +} + +func (c *client) DoRequest(ctx context.Context, streamsLookup *mercury.StreamsLookup, pluginRetryKey string) (mercury.MercuryUpkeepState, mercury.MercuryUpkeepFailureReason, [][]byte, bool, time.Duration, error) { + resultLen := len(streamsLookup.Feeds) + ch := make(chan mercury.MercuryData, resultLen) + if len(streamsLookup.Feeds) == 0 { + return mercury.NoPipelineError, mercury.MercuryUpkeepFailureReasonInvalidRevertDataInput, [][]byte{}, false, 0 * time.Second, fmt.Errorf("invalid revert data input: feed param key %s, time param key %s, feeds %s", streamsLookup.FeedParamKey, streamsLookup.TimeParamKey, streamsLookup.Feeds) + } + resultLen = 1 + go c.multiFeedsRequest(ctx, ch, streamsLookup) + + var reqErr error + var retryInterval time.Duration + results := make([][]byte, len(streamsLookup.Feeds)) + retryable := true + allSuccess := true + state := mercury.NoPipelineError + + for i := 0; i < resultLen; i++ { + m := <-ch + if m.Error != nil { + reqErr = errors.Join(reqErr, m.Error) + retryable = retryable && m.Retryable + allSuccess = false + if m.State != mercury.NoPipelineError { + state = m.State + } + continue + } + results = m.Bytes + } + // only retry when not all successful AND none are not retryable + if retryable && !allSuccess { + retryInterval = mercury.CalculateRetryConfigFn(pluginRetryKey, c.mercuryConfig) + } + // only retry when not all successful AND none are not retryable + return state, mercury.MercuryUpkeepFailureReasonNone, results, retryable && !allSuccess, retryInterval, reqErr +} + +func (c *client) multiFeedsRequest(ctx context.Context, ch chan<- mercury.MercuryData, sl *mercury.StreamsLookup) { + // this won't work bc q.Encode() will encode commas as '%2C' but the server is strictly expecting a comma separated list + //q := url.Values{ + // feedIDs: {strings.Join(sl.Feeds, ",")}, + // timestamp: {sl.Time.String()}, + //} + + params := fmt.Sprintf("%s=%s&%s=%s", mercury.FeedIDs, strings.Join(sl.Feeds, ","), mercury.Timestamp, sl.Time.String()) + batchPathV03 := mercuryBatchPathV03 + if sl.IsMercuryUsingBatchPathV03() { + batchPathV03 = mercuryBatchPathV03BlockNumber + } + reqUrl := fmt.Sprintf("%s%s%s", c.mercuryConfig.Credentials().URL, batchPathV03, params) + + c.lggr.Debugf("request URL for upkeep %s userId %s: %s", sl.UpkeepId.String(), c.mercuryConfig.Credentials().Username, reqUrl) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqUrl, nil) + if err != nil { + ch <- mercury.MercuryData{Index: 0, Error: err, Retryable: false, State: mercury.InvalidMercuryRequest} + return + } + + ts := time.Now().UTC().UnixMilli() + signature := mercury.GenerateHMACFn(http.MethodGet, mercuryBatchPathV03+params, []byte{}, c.mercuryConfig.Credentials().Username, c.mercuryConfig.Credentials().Password, ts) + + req.Header.Set(contentTypeHeader, "application/json") + // username here is often referred to as user id + req.Header.Set(authorizationHeader, c.mercuryConfig.Credentials().Username) + req.Header.Set(timestampHeader, strconv.FormatInt(ts, 10)) + req.Header.Set(signatureHeader, signature) + // mercury will inspect authorization headers above to make sure this user (in automation's context, this node) is eligible to access mercury + // and if it has an automation role. it will then look at this upkeep id to check if it has access to all the requested feeds. + req.Header.Set(upkeepIDHeader, sl.UpkeepId.String()) + + // in the case of multiple retries here, use the last attempt's data + state := mercury.NoPipelineError + retryable := false + sent := false + retryErr := retry.Do( + func() error { + retryable = false + resp, err := c.httpClient.Do(req) + if err != nil { + c.lggr.Warnf("at timestamp %s upkeep %s GET request fails from mercury v0.3: %v", sl.Time.String(), sl.UpkeepId.String(), err) + retryable = true + state = mercury.MercuryFlakyFailure + return err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + retryable = false + state = mercury.InvalidMercuryResponse + return err + } + + c.lggr.Infof("at timestamp %s upkeep %s received status code %d from mercury v0.3", sl.Time.String(), sl.UpkeepId.String(), resp.StatusCode) + switch resp.StatusCode { + case http.StatusUnauthorized: + retryable = false + state = mercury.UpkeepNotAuthorized + return fmt.Errorf("at timestamp %s upkeep %s received status code %d from mercury v0.3, most likely this is caused by unauthorized upkeep", sl.Time.String(), sl.UpkeepId.String(), resp.StatusCode) + case http.StatusBadRequest: + retryable = false + state = mercury.InvalidMercuryRequest + return fmt.Errorf("at timestamp %s upkeep %s received status code %d from mercury v0.3, most likely this is caused by invalid format of timestamp", sl.Time.String(), sl.UpkeepId.String(), resp.StatusCode) + case http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout: + retryable = true + state = mercury.MercuryFlakyFailure + return fmt.Errorf("%d", resp.StatusCode) + case http.StatusPartialContent: + // TODO (AUTO-5044): handle response code 206 entirely with errors field parsing + c.lggr.Warnf("at timestamp %s upkeep %s requested [%s] feeds but mercury v0.3 server returned 206 status, treating it as 404 and retrying", sl.Time.String(), sl.UpkeepId.String(), sl.Feeds) + retryable = true + state = mercury.MercuryFlakyFailure + return fmt.Errorf("%d", http.StatusPartialContent) + case http.StatusOK: + // continue + default: + retryable = false + state = mercury.InvalidMercuryRequest + return fmt.Errorf("at timestamp %s upkeep %s received status code %d from mercury v0.3", sl.Time.String(), sl.UpkeepId.String(), resp.StatusCode) + } + c.lggr.Debugf("at block %s upkeep %s received status code %d from mercury v0.3 with BODY=%s", sl.Time.String(), sl.UpkeepId.String(), resp.StatusCode, hexutil.Encode(body)) + + var response MercuryV03Response + if err := json.Unmarshal(body, &response); err != nil { + c.lggr.Warnf("at timestamp %s upkeep %s failed to unmarshal body to MercuryV03Response from mercury v0.3: %v", sl.Time.String(), sl.UpkeepId.String(), err) + retryable = false + state = mercury.MercuryUnmarshalError + return err + } + + // in v0.3, if some feeds are not available, the server will only return available feeds, but we need to make sure ALL feeds are retrieved before calling user contract + // hence, retry in this case. retry will help when we send a very new timestamp and reports are not yet generated + if len(response.Reports) != len(sl.Feeds) { + var receivedFeeds []string + for _, f := range response.Reports { + receivedFeeds = append(receivedFeeds, f.FeedID) + } + c.lggr.Warnf("at timestamp %s upkeep %s mercury v0.3 server returned 206 status with [%s] reports while we requested [%s] feeds, retrying", sl.Time.String(), sl.UpkeepId.String(), receivedFeeds, sl.Feeds) + retryable = true + state = mercury.MercuryFlakyFailure + return fmt.Errorf("%d", http.StatusNotFound) + } + var reportBytes [][]byte + for _, rsp := range response.Reports { + b, err := hexutil.Decode(rsp.FullReport) + if err != nil { + c.lggr.Warnf("at timestamp %s upkeep %s failed to decode reportBlob %s: %v", sl.Time.String(), sl.UpkeepId.String(), rsp.FullReport, err) + retryable = false + state = mercury.InvalidMercuryResponse + return err + } + reportBytes = append(reportBytes, b) + } + ch <- mercury.MercuryData{ + Index: 0, + Bytes: reportBytes, + Retryable: false, + State: mercury.NoPipelineError, + } + sent = true + return nil + }, + // only retry when the error is 206 Partial Content, 404 Not Found, 500 Internal Server Error, 502 Bad Gateway, 503 Service Unavailable, 504 Gateway Timeout + retry.RetryIf(func(err error) bool { + return err.Error() == fmt.Sprintf("%d", http.StatusPartialContent) || err.Error() == fmt.Sprintf("%d", http.StatusNotFound) || err.Error() == fmt.Sprintf("%d", http.StatusInternalServerError) || err.Error() == fmt.Sprintf("%d", http.StatusBadGateway) || err.Error() == fmt.Sprintf("%d", http.StatusServiceUnavailable) || err.Error() == fmt.Sprintf("%d", http.StatusGatewayTimeout) + }), + retry.Context(ctx), + retry.Delay(retryDelay), + retry.Attempts(totalAttempt), + ) + + if !sent { + ch <- mercury.MercuryData{ + Index: 0, + Bytes: [][]byte{}, + Retryable: retryable, + Error: retryErr, + State: state, + } + } +} diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v03/v03_request_test.go b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v03/v03_request_test.go new file mode 100644 index 00000000000..a66306d31df --- /dev/null +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/v03/v03_request_test.go @@ -0,0 +1,530 @@ +package v03 + +import ( + "bytes" + "context" + "encoding/json" + "io" + "math/big" + "net/http" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/patrickmn/go-cache" + "github.com/stretchr/testify/mock" + + "github.com/smartcontractkit/chainlink/v2/core/logger" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/models" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mocks" + + "github.com/stretchr/testify/assert" + + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury" +) + +const ( + defaultPluginRetryExpiration = 30 * time.Minute + cleanupInterval = 5 * time.Minute +) + +type MockMercuryConfigProvider struct { + cache *cache.Cache + mock.Mock +} + +func NewMockMercuryConfigProvider() *MockMercuryConfigProvider { + return &MockMercuryConfigProvider{ + cache: cache.New(defaultPluginRetryExpiration, cleanupInterval), + } +} + +func (m *MockMercuryConfigProvider) Credentials() *models.MercuryCredentials { + mc := &models.MercuryCredentials{ + LegacyURL: "https://google.old.com", + URL: "https://google.com", + Username: "FakeClientID", + Password: "FakeClientKey", + } + return mc +} + +func (m *MockMercuryConfigProvider) IsUpkeepAllowed(s string) (interface{}, bool) { + args := m.Called(s) + return args.Get(0), args.Bool(1) +} + +func (m *MockMercuryConfigProvider) SetUpkeepAllowed(s string, i interface{}, d time.Duration) { + m.Called(s, i, d) +} + +func (m *MockMercuryConfigProvider) GetPluginRetry(s string) (interface{}, bool) { + if value, found := m.cache.Get(s); found { + return value, true + } + + return nil, false +} + +func (m *MockMercuryConfigProvider) SetPluginRetry(s string, i interface{}, d time.Duration) { + m.cache.Set(s, i, d) +} + +type MockHttpClient struct { + mock.Mock +} + +func (mock *MockHttpClient) Do(req *http.Request) (*http.Response, error) { + args := mock.Called(req) + return args.Get(0).(*http.Response), args.Error(1) +} + +// setups up a client object for tests. +func setupClient(t *testing.T) *client { + lggr := logger.TestLogger(t) + mockHttpClient := new(MockHttpClient) + mercuryConfig := NewMockMercuryConfigProvider() + client := NewClient( + mercuryConfig, + mockHttpClient, + lggr, + ) + return client +} + +func TestV03_DoMercuryRequestV03(t *testing.T) { + upkeepId, _ := new(big.Int).SetString("88786950015966611018675766524283132478093844178961698330929478019253453382042", 10) + + tests := []struct { + name string + lookup *mercury.StreamsLookup + mockHttpStatusCode int + mockChainlinkBlobs []string + pluginRetryKey string + expectedValues [][]byte + expectedRetryable bool + expectedRetryInterval time.Duration + expectedError error + state mercury.MercuryUpkeepState + reason mercury.MercuryUpkeepFailureReason + }{ + { + name: "success v0.3", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIDs, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(25880526), + ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, + }, + UpkeepId: upkeepId, + }, + mockHttpStatusCode: http.StatusOK, + mockChainlinkBlobs: []string{"0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000000081401000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000289ad8d367000000000000000000000000000000000000000000000000000000289acf0b38000000000000000000000000000000000000000000000000000000289b3da40000000000000000000000000000000000000000000000000000000000018ae7ce74d9fa252a8983976eab600dc7590c778d04813430841bc6e765c34cd81a168d00000000000000000000000000000000000000000000000000000000018ae7cb0000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000000000000260412b94e525ca6cedc9f544fd86f77606d52fe731a5d069dbe836a8bfc0fb8c911963b0ae7a14971f3b4621bffb802ef0605392b9a6c89c7fab1df8633a5ade00000000000000000000000000000000000000000000000000000000000000024500c2f521f83fba5efc2bf3effaaedde43d0a4adff785c1213b712a3aed0d8157642a84324db0cf9695ebd27708d4608eb0337e0dd87b0e43f0fa70c700d911"}, + expectedValues: [][]byte{{0, 6, 109, 252, 209, 237, 45, 149, 177, 140, 148, 141, 188, 91, 214, 76, 104, 122, 254, 147, 228, 202, 125, 102, 61, 222, 193, 76, 32, 9, 10, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 128, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 69, 84, 72, 45, 85, 83, 68, 45, 65, 82, 66, 73, 84, 82, 85, 77, 45, 84, 69, 83, 84, 78, 69, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 137, 28, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 154, 216, 211, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 154, 207, 11, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 155, 61, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 138, 231, 206, 116, 217, 250, 37, 42, 137, 131, 151, 110, 171, 96, 13, 199, 89, 12, 119, 141, 4, 129, 52, 48, 132, 27, 198, 231, 101, 195, 76, 216, 26, 22, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 138, 231, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 137, 28, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 96, 65, 43, 148, 229, 37, 202, 108, 237, 201, 245, 68, 253, 134, 247, 118, 6, 213, 47, 231, 49, 165, 208, 105, 219, 232, 54, 168, 191, 192, 251, 140, 145, 25, 99, 176, 174, 122, 20, 151, 31, 59, 70, 33, 191, 251, 128, 46, 240, 96, 83, 146, 185, 166, 200, 156, 127, 171, 29, 248, 99, 58, 90, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 69, 0, 194, 245, 33, 248, 63, 186, 94, 252, 43, 243, 239, 250, 174, 221, 228, 61, 10, 74, 223, 247, 133, 193, 33, 59, 113, 42, 58, 237, 13, 129, 87, 100, 42, 132, 50, 77, 176, 207, 150, 149, 235, 210, 119, 8, 212, 96, 142, 176, 51, 126, 13, 216, 123, 14, 67, 240, 250, 112, 199, 0, 217, 17}}, + expectedRetryable: false, + expectedError: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := setupClient(t) + hc := mocks.NewHttpClient(t) + + mr := MercuryV03Response{} + for i, blob := range tt.mockChainlinkBlobs { + r := MercuryV03Report{ + FeedID: tt.lookup.Feeds[i], + ValidFromTimestamp: 0, + ObservationsTimestamp: 0, + FullReport: blob, + } + mr.Reports = append(mr.Reports, r) + } + + b, err := json.Marshal(mr) + assert.Nil(t, err) + resp := &http.Response{ + StatusCode: tt.mockHttpStatusCode, + Body: io.NopCloser(bytes.NewReader(b)), + } + if tt.expectedError != nil && tt.expectedRetryable { + hc.On("Do", mock.Anything).Return(resp, nil).Times(totalAttempt) + } else { + hc.On("Do", mock.Anything).Return(resp, nil).Once() + } + c.httpClient = hc + + state, reason, values, retryable, retryInterval, reqErr := c.DoRequest(context.Background(), tt.lookup, tt.pluginRetryKey) + + assert.Equal(t, tt.expectedValues, values) + assert.Equal(t, tt.expectedRetryable, retryable) + assert.Equal(t, tt.expectedRetryInterval, retryInterval) + assert.Equal(t, tt.state, state) + assert.Equal(t, tt.reason, reason) + if tt.expectedError != nil { + assert.Equal(t, tt.expectedError.Error(), reqErr.Error()) + } + }) + } +} + +func TestV03_MultiFeedRequest(t *testing.T) { + upkeepId := big.NewInt(123456789) + tests := []struct { + name string + lookup *mercury.StreamsLookup + statusCode int + lastStatusCode int + pluginRetries int + pluginRetryKey string + retryNumber int + retryable bool + errorMessage string + firstResponse *MercuryV03Response + response *MercuryV03Response + }{ + { + name: "success - mercury responds in the first try", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIDs, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.Timestamp, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + response: &MercuryV03Response{ + Reports: []MercuryV03Report{ + { + FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", + ValidFromTimestamp: 123456, + ObservationsTimestamp: 123456, + FullReport: "0xab2123dc00000012", + }, + { + FeedID: "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000", + ValidFromTimestamp: 123458, + ObservationsTimestamp: 123458, + FullReport: "0xab2123dc00000016", + }, + }, + }, + statusCode: http.StatusOK, + }, + { + name: "success - mercury responds in the first try with blocknumber", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIDs, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.BlockNumber, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + response: &MercuryV03Response{ + Reports: []MercuryV03Report{ + { + FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", + ValidFromTimestamp: 123456, + ObservationsTimestamp: 123456, + FullReport: "0xab2123dc00000012", + }, + { + FeedID: "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000", + ValidFromTimestamp: 123458, + ObservationsTimestamp: 123458, + FullReport: "0xab2123dc00000016", + }, + }, + }, + statusCode: http.StatusOK, + }, + { + name: "success - retry 206", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIDs, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.Timestamp, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + firstResponse: &MercuryV03Response{ + Reports: []MercuryV03Report{ + { + FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", + ValidFromTimestamp: 123456, + ObservationsTimestamp: 123456, + FullReport: "0xab2123dc00000012", + }, + }, + }, + response: &MercuryV03Response{ + Reports: []MercuryV03Report{ + { + FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", + ValidFromTimestamp: 123456, + ObservationsTimestamp: 123456, + FullReport: "0xab2123dc00000012", + }, + { + FeedID: "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000", + ValidFromTimestamp: 123458, + ObservationsTimestamp: 123458, + FullReport: "0xab2123dc00000019", + }, + }, + }, + retryNumber: 1, + statusCode: http.StatusPartialContent, + lastStatusCode: http.StatusOK, + }, + { + name: "success - retry for 500", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIDs, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.Timestamp, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + retryNumber: 2, + statusCode: http.StatusInternalServerError, + lastStatusCode: http.StatusOK, + response: &MercuryV03Response{ + Reports: []MercuryV03Report{ + { + FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", + ValidFromTimestamp: 123456, + ObservationsTimestamp: 123456, + FullReport: "0xab2123dc00000012", + }, + { + FeedID: "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000", + ValidFromTimestamp: 123458, + ObservationsTimestamp: 123458, + FullReport: "0xab2123dc00000019", + }, + }, + }, + }, + { + name: "failure - fail to decode reportBlob", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIDs, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.Timestamp, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + response: &MercuryV03Response{ + Reports: []MercuryV03Report{ + { + FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", + ValidFromTimestamp: 123456, + ObservationsTimestamp: 123456, + FullReport: "qerwiu", // invalid hex blob + }, + { + FeedID: "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000", + ValidFromTimestamp: 123458, + ObservationsTimestamp: 123458, + FullReport: "0xab2123dc00000016", + }, + }, + }, + statusCode: http.StatusOK, + retryable: false, + errorMessage: "All attempts fail:\n#1: hex string without 0x prefix", + }, + { + name: "failure - returns retryable with 1s plugin retry interval", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIDs, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.Timestamp, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + retryNumber: totalAttempt, + statusCode: http.StatusInternalServerError, + retryable: true, + errorMessage: "All attempts fail:\n#1: 500\n#2: 500\n#3: 500", + }, + { + name: "failure - returns retryable with 5s plugin retry interval", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIDs, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.Timestamp, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + pluginRetries: 6, + retryNumber: totalAttempt, + statusCode: http.StatusInternalServerError, + retryable: true, + errorMessage: "All attempts fail:\n#1: 500\n#2: 500\n#3: 500", + }, + { + name: "failure - returns retryable and then non-retryable", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIDs, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.Timestamp, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + retryNumber: 1, + statusCode: http.StatusInternalServerError, + lastStatusCode: http.StatusUnauthorized, + errorMessage: "All attempts fail:\n#1: 500\n#2: at timestamp 123456 upkeep 123456789 received status code 401 from mercury v0.3, most likely this is caused by unauthorized upkeep", + }, + { + name: "failure - returns status code 422 not retryable", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIDs, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.Timestamp, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + statusCode: http.StatusUnprocessableEntity, + errorMessage: "All attempts fail:\n#1: at timestamp 123456 upkeep 123456789 received status code 422 from mercury v0.3", + }, + { + name: "success - retry when reports length does not match feeds length", + lookup: &mercury.StreamsLookup{ + StreamsLookupError: &mercury.StreamsLookupError{ + FeedParamKey: mercury.FeedIDs, + Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, + TimeParamKey: mercury.Timestamp, + Time: big.NewInt(123456), + }, + UpkeepId: upkeepId, + }, + firstResponse: &MercuryV03Response{ + Reports: []MercuryV03Report{ + { + FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", + ValidFromTimestamp: 123456, + ObservationsTimestamp: 123456, + FullReport: "0xab2123dc00000012", + }, + }, + }, + response: &MercuryV03Response{ + Reports: []MercuryV03Report{ + { + FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", + ValidFromTimestamp: 123456, + ObservationsTimestamp: 123456, + FullReport: "0xab2123dc00000012", + }, + { + FeedID: "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000", + ValidFromTimestamp: 123458, + ObservationsTimestamp: 123458, + FullReport: "0xab2123dc00000019", + }, + }, + }, + retryNumber: 1, + statusCode: http.StatusOK, + lastStatusCode: http.StatusOK, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := setupClient(t) + if tt.pluginRetries != 0 { + c.mercuryConfig.SetPluginRetry(tt.pluginRetryKey, tt.pluginRetries, cache.DefaultExpiration) + } + + hc := new(MockHttpClient) + b, err := json.Marshal(tt.response) + assert.Nil(t, err) + + if tt.retryNumber == 0 { + resp := &http.Response{ + StatusCode: tt.statusCode, + Body: io.NopCloser(bytes.NewReader(b)), + } + hc.On("Do", mock.Anything).Return(resp, nil).Once() + } else if tt.retryNumber < totalAttempt { + if tt.firstResponse != nil && tt.response != nil { + b0, err := json.Marshal(tt.firstResponse) + assert.Nil(t, err) + resp0 := &http.Response{ + StatusCode: tt.statusCode, + Body: io.NopCloser(bytes.NewReader(b0)), + } + b1, err := json.Marshal(tt.response) + assert.Nil(t, err) + resp1 := &http.Response{ + StatusCode: tt.lastStatusCode, + Body: io.NopCloser(bytes.NewReader(b1)), + } + hc.On("Do", mock.Anything).Return(resp0, nil).Once().On("Do", mock.Anything).Return(resp1, nil).Once() + } else { + retryResp := &http.Response{ + StatusCode: tt.statusCode, + Body: io.NopCloser(bytes.NewReader(b)), + } + hc.On("Do", mock.Anything).Return(retryResp, nil).Times(tt.retryNumber) + + resp := &http.Response{ + StatusCode: tt.lastStatusCode, + Body: io.NopCloser(bytes.NewReader(b)), + } + hc.On("Do", mock.Anything).Return(resp, nil).Once() + } + } else { + resp := &http.Response{ + StatusCode: tt.statusCode, + Body: io.NopCloser(bytes.NewReader(b)), + } + hc.On("Do", mock.Anything).Return(resp, nil).Times(tt.retryNumber) + } + c.httpClient = hc + + ch := make(chan mercury.MercuryData, 1) + c.multiFeedsRequest(context.Background(), ch, tt.lookup) + + m := <-ch + assert.Equal(t, 0, m.Index) + assert.Equal(t, tt.retryable, m.Retryable) + if tt.retryNumber >= totalAttempt || tt.errorMessage != "" { + assert.Equal(t, tt.errorMessage, m.Error.Error()) + assert.Equal(t, [][]byte{}, m.Bytes) + } else { + assert.Nil(t, m.Error) + var reports [][]byte + for _, rsp := range tt.response.Reports { + b, _ := hexutil.Decode(rsp.FullReport) + reports = append(reports, b) + } + assert.Equal(t, reports, m.Bytes) + } + }) + } +} diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/registry.go b/core/services/ocr2/plugins/ocr2keeper/evm21/registry.go index 73e2bc0a9c0..fec23909ae1 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evm21/registry.go +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/registry.go @@ -17,9 +17,9 @@ import ( "github.com/pkg/errors" "go.uber.org/multierr" - ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/streams" - "github.com/smartcontractkit/chainlink-relay/pkg/services" + ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" "github.com/smartcontractkit/chainlink/v2/core/chains/evm" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" @@ -42,8 +42,6 @@ const ( // cleanupInterval decides when the expired items in cache will be deleted. cleanupInterval = 5 * time.Minute logTriggerRefreshBatchSize = 32 - totalFastPluginRetries = 5 - totalMediumPluginRetries = 10 ) var ( @@ -89,30 +87,34 @@ func NewEvmRegistry( blockSub *BlockSubscriber, finalityDepth uint32, ) *EvmRegistry { + mercuryConfig := &MercuryConfig{ + Cred: mc, + Abi: core.StreamsCompatibleABI, + AllowListCache: cache.New(defaultAllowListExpiration, cleanupInterval), + pluginRetryCache: cache.New(defaultPluginRetryExpiration, cleanupInterval), + } + + hc := http.DefaultClient + return &EvmRegistry{ - ctx: context.Background(), - threadCtrl: utils.NewThreadControl(), - lggr: lggr.Named(RegistryServiceName), - poller: client.LogPoller(), - addr: addr, - client: client.Client(), - logProcessed: make(map[string]bool), - registry: registry, - abi: core.RegistryABI, - active: al, - packer: packer, - headFunc: func(ocr2keepers.BlockKey) {}, - chLog: make(chan logpoller.Log, 1000), - mercury: &MercuryConfig{ - cred: mc, - abi: core.StreamsCompatibleABI, - allowListCache: cache.New(defaultAllowListExpiration, cleanupInterval), - pluginRetryCache: cache.New(defaultPluginRetryExpiration, cleanupInterval), - }, - hc: http.DefaultClient, + ctx: context.Background(), + threadCtrl: utils.NewThreadControl(), + lggr: lggr.Named(RegistryServiceName), + poller: client.LogPoller(), + addr: addr, + client: client.Client(), + logProcessed: make(map[string]bool), + registry: registry, + abi: core.RegistryABI, + active: al, + packer: packer, + headFunc: func(ocr2keepers.BlockKey) {}, + chLog: make(chan logpoller.Log, 1000), + hc: hc, logEventProvider: logEventProvider, bs: blockSub, finalityDepth: finalityDepth, + streams: streams.NewStreamsLookup(packer, mercuryConfig, blockSub, registry, lggr), } } @@ -127,16 +129,45 @@ var upkeepStateEvents = []common.Hash{ } type MercuryConfig struct { - cred *models.MercuryCredentials - abi abi.ABI - // allowListCache stores the upkeeps privileges. In 2.1, this only includes a JSON bytes for allowed to use mercury - allowListCache *cache.Cache - + Cred *models.MercuryCredentials + Abi abi.ABI + // AllowListCache stores the upkeeps privileges. In 2.1, this only includes a JSON bytes for allowed to use mercury + AllowListCache *cache.Cache pluginRetryCache *cache.Cache } +func NewMercuryConfig(credentials *models.MercuryCredentials, abi abi.ABI) *MercuryConfig { + return &MercuryConfig{ + Cred: credentials, + Abi: abi, + AllowListCache: cache.New(defaultPluginRetryExpiration, cleanupInterval), + pluginRetryCache: cache.New(defaultPluginRetryExpiration, cleanupInterval), + } +} + +// TODO since Cred is now exposed, we should remove this method and use Cred directly +func (c *MercuryConfig) Credentials() *models.MercuryCredentials { + return c.Cred +} + +func (c *MercuryConfig) IsUpkeepAllowed(k string) (interface{}, bool) { + return c.AllowListCache.Get(k) +} + +func (c *MercuryConfig) SetUpkeepAllowed(k string, v interface{}, d time.Duration) { + c.AllowListCache.Set(k, v, d) +} + +func (c *MercuryConfig) GetPluginRetry(k string) (interface{}, bool) { + return c.pluginRetryCache.Get(k) +} + +func (c *MercuryConfig) SetPluginRetry(k string, v interface{}, d time.Duration) { + c.pluginRetryCache.Set(k, v, d) +} + type EvmRegistry struct { - services.StateMachine + utils.StartStopOnce threadCtrl utils.ThreadControl lggr logger.Logger poller logpoller.LogPoller @@ -158,6 +189,7 @@ type EvmRegistry struct { bs *BlockSubscriber logEventProvider logprovider.LogEventProvider finalityDepth uint32 + streams streams.Lookup } func (r *EvmRegistry) Name() string { diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go b/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go index d3530994702..5242734c5d3 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline.go @@ -67,7 +67,7 @@ func (r *EvmRegistry) doCheck(ctx context.Context, keys []ocr2keepers.UpkeepPayl return } - upkeepResults = r.streamsLookup(ctx, upkeepResults) + upkeepResults = r.streams.Lookup(ctx, upkeepResults) upkeepResults, err = r.simulatePerformUpkeeps(ctx, upkeepResults) if err != nil { @@ -101,7 +101,7 @@ func (r *EvmRegistry) getBlockHash(blockNumber *big.Int) (common.Hash, error) { } // verifyCheckBlock checks that the check block and hash are valid, returns the pipeline execution state and retryable -func (r *EvmRegistry) verifyCheckBlock(_ context.Context, checkBlock, upkeepId *big.Int, checkHash common.Hash) (state encoding.PipelineExecutionState, retryable bool) { +func (r *EvmRegistry) verifyCheckBlock(_ context.Context, checkBlock, upkeepId *big.Int, checkHash common.Hash) (state uint8, retryable bool) { // verify check block number and hash are valid h, ok := r.bs.queryBlocksMap(checkBlock.Int64()) // if this block number/hash combo exists in block subscriber, this check block and hash still exist on chain and are valid @@ -124,7 +124,7 @@ func (r *EvmRegistry) verifyCheckBlock(_ context.Context, checkBlock, upkeepId * } // verifyLogExists checks that the log still exists on chain, returns failure reason, pipeline error, and retryable -func (r *EvmRegistry) verifyLogExists(upkeepId *big.Int, p ocr2keepers.UpkeepPayload) (encoding.UpkeepFailureReason, encoding.PipelineExecutionState, bool) { +func (r *EvmRegistry) verifyLogExists(upkeepId *big.Int, p ocr2keepers.UpkeepPayload) (uint8, uint8, bool) { logBlockNumber := int64(p.Trigger.LogTriggerExtension.BlockNumber) logBlockHash := common.BytesToHash(p.Trigger.LogTriggerExtension.BlockHash[:]) checkBlockHash := common.BytesToHash(p.Trigger.BlockHash[:]) diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline_test.go b/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline_test.go index 5ea2bdc6670..527928f6522 100644 --- a/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline_test.go +++ b/core/services/ocr2/plugins/ocr2keeper/evm21/registry_check_pipeline_test.go @@ -4,12 +4,23 @@ import ( "context" "fmt" "math/big" + "strings" "sync/atomic" "testing" + "github.com/stretchr/testify/require" + + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/patrickmn/go-cache" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rpc" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/i_keeper_registry_master_wrapper_2_1" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/streams_lookup_compatible_interface" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/models" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mocks" + ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -81,7 +92,7 @@ func TestRegistry_VerifyCheckBlock(t *testing.T) { payload ocr2keepers.UpkeepPayload blocks map[int64]string poller logpoller.LogPoller - state encoding.PipelineExecutionState + state uint8 retryable bool makeEthCall bool }{ @@ -239,8 +250,8 @@ func TestRegistry_VerifyLogExists(t *testing.T) { payload ocr2keepers.UpkeepPayload blocks map[int64]string makeEthCall bool - reason encoding.UpkeepFailureReason - state encoding.PipelineExecutionState + reason uint8 + state uint8 retryable bool ethCallErr error receipt *types.Receipt @@ -645,5 +656,44 @@ func TestRegistry_SimulatePerformUpkeeps(t *testing.T) { assert.Equal(t, tc.err, err) }) } +} +// setups up an evm registry for tests. +func setupEVMRegistry(t *testing.T) *EvmRegistry { + lggr := logger.TestLogger(t) + addr := common.HexToAddress("0x6cA639822c6C241Fa9A7A6b5032F6F7F1C513CAD") + keeperRegistryABI, err := abi.JSON(strings.NewReader(i_keeper_registry_master_wrapper_2_1.IKeeperRegistryMasterABI)) + require.Nil(t, err, "need registry abi") + streamsLookupCompatibleABI, err := abi.JSON(strings.NewReader(streams_lookup_compatible_interface.StreamsLookupCompatibleInterfaceABI)) + require.Nil(t, err, "need mercury abi") + var logPoller logpoller.LogPoller + mockReg := mocks.NewRegistry(t) + mockHttpClient := mocks.NewHttpClient(t) + client := evmClientMocks.NewClient(t) + + r := &EvmRegistry{ + lggr: lggr, + poller: logPoller, + addr: addr, + client: client, + logProcessed: make(map[string]bool), + registry: mockReg, + abi: keeperRegistryABI, + active: NewActiveUpkeepList(), + packer: encoding.NewAbiPacker(), + headFunc: func(ocr2keepers.BlockKey) {}, + chLog: make(chan logpoller.Log, 1000), + mercury: &MercuryConfig{ + Cred: &models.MercuryCredentials{ + LegacyURL: "https://google.old.com", + URL: "https://google.com", + Username: "FakeClientID", + Password: "FakeClientKey", + }, + Abi: streamsLookupCompatibleABI, + AllowListCache: cache.New(defaultAllowListExpiration, cleanupInterval), + }, + hc: mockHttpClient, + } + return r } diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/streams_lookup.go b/core/services/ocr2/plugins/ocr2keeper/evm21/streams_lookup.go deleted file mode 100644 index 660550afe97..00000000000 --- a/core/services/ocr2/plugins/ocr2keeper/evm21/streams_lookup.go +++ /dev/null @@ -1,630 +0,0 @@ -package evm - -import ( - "context" - "crypto/hmac" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "math/big" - "net/http" - "net/url" - "strconv" - "strings" - "sync" - "time" - - "github.com/avast/retry-go/v4" - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/patrickmn/go-cache" - ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" - - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/encoding" -) - -const ( - applicationJson = "application/json" - blockNumber = "blockNumber" // valid for v0.2 - feedIDs = "feedIDs" // valid for v0.3 - feedIdHex = "feedIdHex" // valid for v0.2 - headerAuthorization = "Authorization" - headerContentType = "Content-Type" - headerTimestamp = "X-Authorization-Timestamp" - headerSignature = "X-Authorization-Signature-SHA256" - headerUpkeepId = "X-Authorization-Upkeep-Id" - mercuryPathV02 = "/client?" // only used to access mercury v0.2 server - mercuryBatchPathV03 = "/api/v1/reports/bulk?" // only used to access mercury v0.3 server - mercuryBatchPathV03BlockNumber = "/api/v1gmx/reports/bulk?" // only used to access mercury v0.3 server with blockNumber - retryDelay = 500 * time.Millisecond - timestamp = "timestamp" // valid for v0.3 - totalAttempt = 3 -) - -type StreamsLookup struct { - *encoding.StreamsLookupError - upkeepId *big.Int - block uint64 -} - -// MercuryV02Response represents a JSON structure used by Mercury v0.2 -type MercuryV02Response struct { - ChainlinkBlob string `json:"chainlinkBlob"` -} - -// MercuryV03Response represents a JSON structure used by Mercury v0.3 -type MercuryV03Response struct { - Reports []MercuryV03Report `json:"reports"` -} - -type MercuryV03Report struct { - FeedID string `json:"feedID"` // feed id in hex encoded - ValidFromTimestamp uint32 `json:"validFromTimestamp"` - ObservationsTimestamp uint32 `json:"observationsTimestamp"` - FullReport string `json:"fullReport"` // the actual hex encoded mercury report of this feed, can be sent to verifier -} - -type MercuryData struct { - Index int - Error error - Retryable bool - Bytes [][]byte - State encoding.PipelineExecutionState -} - -// UpkeepPrivilegeConfig represents the administrative offchain config for each upkeep. It can be set by s_upkeepPrivilegeManager -// role on the registry. Upkeeps allowed to use Mercury server will have this set to true. -type UpkeepPrivilegeConfig struct { - MercuryEnabled bool `json:"mercuryEnabled"` -} - -// streamsLookup looks through check upkeep results looking for any that need off chain lookup -func (r *EvmRegistry) streamsLookup(ctx context.Context, checkResults []ocr2keepers.CheckResult) []ocr2keepers.CheckResult { - lggr := r.lggr.With("where", "StreamsLookup") - lookups := map[int]*StreamsLookup{} - for i, res := range checkResults { - if res.IneligibilityReason != uint8(encoding.UpkeepFailureReasonTargetCheckReverted) { - // Streams Lookup only works when upkeep target check reverts - continue - } - - block := big.NewInt(int64(res.Trigger.BlockNumber)) - upkeepId := res.UpkeepID - - // Try to decode the revert error into streams lookup format. User upkeeps can revert with any reason, see if they - // tried to call mercury - lggr.Infof("at block %d upkeep %s trying to DecodeStreamsLookupRequest performData=%s", block, upkeepId, hexutil.Encode(checkResults[i].PerformData)) - streamsLookupErr, err := r.packer.DecodeStreamsLookupRequest(res.PerformData) - if err != nil { - lggr.Debugf("at block %d upkeep %s DecodeStreamsLookupRequest failed: %v", block, upkeepId, err) - // user contract did not revert with StreamsLookup error - continue - } - l := &StreamsLookup{StreamsLookupError: streamsLookupErr} - if r.mercury.cred == nil { - lggr.Errorf("at block %d upkeep %s tries to access mercury server but mercury credential is not configured", block, upkeepId) - continue - } - - if len(l.Feeds) == 0 { - checkResults[i].IneligibilityReason = uint8(encoding.UpkeepFailureReasonInvalidRevertDataInput) - lggr.Debugf("at block %s upkeep %s has empty feeds array", block, upkeepId) - continue - } - // mercury permission checking for v0.3 is done by mercury server - if l.FeedParamKey == feedIdHex && l.TimeParamKey == blockNumber { - // check permission on the registry for mercury v0.2 - opts := r.buildCallOpts(ctx, block) - state, reason, retryable, allowed, err := r.allowedToUseMercury(opts, upkeepId.BigInt()) - if err != nil { - lggr.Warnf("at block %s upkeep %s failed to query mercury allow list: %s", block, upkeepId, err) - checkResults[i].PipelineExecutionState = uint8(state) - checkResults[i].IneligibilityReason = uint8(reason) - checkResults[i].Retryable = retryable - continue - } - - if !allowed { - lggr.Debugf("at block %d upkeep %s NOT allowed to query Mercury server", block, upkeepId) - checkResults[i].IneligibilityReason = uint8(encoding.UpkeepFailureReasonMercuryAccessNotAllowed) - continue - } - } else if l.FeedParamKey != feedIDs { - // if mercury version cannot be determined, set failure reason - lggr.Debugf("at block %d upkeep %s NOT allowed to query Mercury server", block, upkeepId) - checkResults[i].IneligibilityReason = uint8(encoding.UpkeepFailureReasonInvalidRevertDataInput) - continue - } - - l.upkeepId = upkeepId.BigInt() - // the block here is exclusively used to call checkCallback at this block, not to be confused with the block number - // in the revert for mercury v0.2, which is denoted by time in the struct bc starting from v0.3, only timestamp will be supported - l.block = uint64(block.Int64()) - lggr.Infof("at block %d upkeep %s DecodeStreamsLookupRequest feedKey=%s timeKey=%s feeds=%v time=%s extraData=%s", block, upkeepId, l.FeedParamKey, l.TimeParamKey, l.Feeds, l.Time, hexutil.Encode(l.ExtraData)) - lookups[i] = l - } - - var wg sync.WaitGroup - for i, lookup := range lookups { - wg.Add(1) - go r.doLookup(ctx, &wg, lookup, i, checkResults, lggr) - } - wg.Wait() - - // don't surface error to plugin bc StreamsLookup process should be self-contained. - return checkResults -} - -func (r *EvmRegistry) doLookup(ctx context.Context, wg *sync.WaitGroup, lookup *StreamsLookup, i int, checkResults []ocr2keepers.CheckResult, lggr logger.Logger) { - defer wg.Done() - - state, reason, values, retryable, ri, err := r.doMercuryRequest(ctx, lookup, generatePluginRetryKey(checkResults[i].WorkID, lookup.block), lggr) - if err != nil { - lggr.Errorf("upkeep %s retryable %v retryInterval %s doMercuryRequest: %s", lookup.upkeepId, retryable, ri, err.Error()) - checkResults[i].Retryable = retryable - checkResults[i].RetryInterval = ri - checkResults[i].PipelineExecutionState = uint8(state) - checkResults[i].IneligibilityReason = uint8(reason) - return - } - - for j, v := range values { - lggr.Infof("upkeep %s doMercuryRequest values[%d]: %s", lookup.upkeepId, j, hexutil.Encode(v)) - } - - state, retryable, mercuryBytes, err := r.checkCallback(ctx, values, lookup) - if err != nil { - lggr.Errorf("at block %d upkeep %s checkCallback err: %s", lookup.block, lookup.upkeepId, err.Error()) - checkResults[i].Retryable = retryable - checkResults[i].PipelineExecutionState = uint8(state) - return - } - lggr.Infof("checkCallback mercuryBytes=%s", hexutil.Encode(mercuryBytes)) - - state, needed, performData, failureReason, _, err := r.packer.UnpackCheckCallbackResult(mercuryBytes) - if err != nil { - lggr.Errorf("at block %d upkeep %s UnpackCheckCallbackResult err: %s", lookup.block, lookup.upkeepId, err.Error()) - checkResults[i].PipelineExecutionState = uint8(state) - return - } - - if failureReason == uint8(encoding.UpkeepFailureReasonMercuryCallbackReverted) { - checkResults[i].IneligibilityReason = uint8(encoding.UpkeepFailureReasonMercuryCallbackReverted) - lggr.Debugf("at block %d upkeep %s mercury callback reverts", lookup.block, lookup.upkeepId) - return - } - - if !needed { - checkResults[i].IneligibilityReason = uint8(encoding.UpkeepFailureReasonUpkeepNotNeeded) - lggr.Debugf("at block %d upkeep %s callback reports upkeep not needed", lookup.block, lookup.upkeepId) - return - } - - checkResults[i].IneligibilityReason = uint8(encoding.UpkeepFailureReasonNone) - checkResults[i].Eligible = true - checkResults[i].PerformData = performData - lggr.Infof("at block %d upkeep %s successful with perform data: %s", lookup.block, lookup.upkeepId, hexutil.Encode(performData)) -} - -// allowedToUseMercury retrieves upkeep's administrative offchain config and decode a mercuryEnabled bool to indicate if -// this upkeep is allowed to use Mercury service. -func (r *EvmRegistry) allowedToUseMercury(opts *bind.CallOpts, upkeepId *big.Int) (state encoding.PipelineExecutionState, reason encoding.UpkeepFailureReason, retryable bool, allow bool, err error) { - allowed, ok := r.mercury.allowListCache.Get(upkeepId.String()) - if ok { - return encoding.NoPipelineError, encoding.UpkeepFailureReasonNone, false, allowed.(bool), nil - } - - payload, err := r.packer.PackGetUpkeepPrivilegeConfig(upkeepId) - if err != nil { - // pack error, no retryable - r.lggr.Warnf("failed to pack getUpkeepPrivilegeConfig data for upkeepId %s: %s", upkeepId, err) - - return encoding.PackUnpackDecodeFailed, encoding.UpkeepFailureReasonNone, false, false, fmt.Errorf("failed to pack upkeepId: %w", err) - } - - var resultBytes hexutil.Bytes - args := map[string]interface{}{ - "to": r.addr.Hex(), - "data": hexutil.Bytes(payload), - } - - // call checkCallback function at the block which OCR3 has agreed upon - err = r.client.CallContext(opts.Context, &resultBytes, "eth_call", args, hexutil.EncodeBig(opts.BlockNumber)) - if err != nil { - return encoding.RpcFlakyFailure, encoding.UpkeepFailureReasonNone, true, false, fmt.Errorf("failed to get upkeep privilege config: %v", err) - } - - cfg, err := r.packer.UnpackGetUpkeepPrivilegeConfig(resultBytes) - if err != nil { - return encoding.PackUnpackDecodeFailed, encoding.UpkeepFailureReasonNone, false, false, fmt.Errorf("failed to get upkeep privilege config: %v", err) - } - - if len(cfg) == 0 { - r.mercury.allowListCache.Set(upkeepId.String(), false, cache.DefaultExpiration) - return encoding.NoPipelineError, encoding.UpkeepFailureReasonMercuryAccessNotAllowed, false, false, fmt.Errorf("upkeep privilege config is empty") - } - - var privilegeConfig UpkeepPrivilegeConfig - if err := json.Unmarshal(cfg, &privilegeConfig); err != nil { - return encoding.MercuryUnmarshalError, encoding.UpkeepFailureReasonNone, false, false, fmt.Errorf("failed to unmarshal privilege config: %v", err) - } - - r.mercury.allowListCache.Set(upkeepId.String(), privilegeConfig.MercuryEnabled, cache.DefaultExpiration) - - return encoding.NoPipelineError, encoding.UpkeepFailureReasonNone, false, privilegeConfig.MercuryEnabled, nil -} - -func (r *EvmRegistry) checkCallback(ctx context.Context, values [][]byte, lookup *StreamsLookup) (encoding.PipelineExecutionState, bool, hexutil.Bytes, error) { - payload, err := r.abi.Pack("checkCallback", lookup.upkeepId, values, lookup.ExtraData) - if err != nil { - return encoding.PackUnpackDecodeFailed, false, nil, err - } - - var b hexutil.Bytes - args := map[string]interface{}{ - "to": r.addr.Hex(), - "data": hexutil.Bytes(payload), - } - - // call checkCallback function at the block which OCR3 has agreed upon - err = r.client.CallContext(ctx, &b, "eth_call", args, hexutil.EncodeUint64(lookup.block)) - if err != nil { - return encoding.RpcFlakyFailure, true, nil, err - } - return encoding.NoPipelineError, false, b, nil -} - -// doMercuryRequest sends requests to Mercury API to retrieve mercury data. -func (r *EvmRegistry) doMercuryRequest(ctx context.Context, sl *StreamsLookup, prk string, lggr logger.Logger) (encoding.PipelineExecutionState, encoding.UpkeepFailureReason, [][]byte, bool, time.Duration, error) { - var isMercuryV03 bool - resultLen := len(sl.Feeds) - ch := make(chan MercuryData, resultLen) - if len(sl.Feeds) == 0 { - return encoding.NoPipelineError, encoding.UpkeepFailureReasonInvalidRevertDataInput, [][]byte{}, false, 0 * time.Second, fmt.Errorf("invalid revert data input: feed param key %s, time param key %s, feeds %s", sl.FeedParamKey, sl.TimeParamKey, sl.Feeds) - } - if sl.FeedParamKey == feedIdHex && sl.TimeParamKey == blockNumber { - // only mercury v0.2 - for i := range sl.Feeds { - go r.singleFeedRequest(ctx, ch, i, sl, lggr) - } - } else if sl.FeedParamKey == feedIDs { - // only mercury v0.3 - resultLen = 1 - isMercuryV03 = true - ch = make(chan MercuryData, resultLen) - go r.multiFeedsRequest(ctx, ch, sl, lggr) - } else { - return encoding.NoPipelineError, encoding.UpkeepFailureReasonInvalidRevertDataInput, [][]byte{}, false, 0 * time.Second, fmt.Errorf("invalid revert data input: feed param key %s, time param key %s, feeds %s", sl.FeedParamKey, sl.TimeParamKey, sl.Feeds) - } - - var reqErr error - var ri time.Duration - results := make([][]byte, len(sl.Feeds)) - retryable := true - allSuccess := true - // in v0.2, use the last execution error as the state, if no execution errors, state will be no error - state := encoding.NoPipelineError - for i := 0; i < resultLen; i++ { - m := <-ch - if m.Error != nil { - reqErr = errors.Join(reqErr, m.Error) - retryable = retryable && m.Retryable - allSuccess = false - if m.State != encoding.NoPipelineError { - state = m.State - } - continue - } - if isMercuryV03 { - results = m.Bytes - } else { - results[m.Index] = m.Bytes[0] - } - } - if retryable && !allSuccess { - ri = r.calculateRetryConfig(prk) - } - // only retry when not all successful AND none are not retryable - return state, encoding.UpkeepFailureReasonNone, results, retryable && !allSuccess, ri, reqErr -} - -// singleFeedRequest sends a v0.2 Mercury request for a single feed report. -func (r *EvmRegistry) singleFeedRequest(ctx context.Context, ch chan<- MercuryData, index int, sl *StreamsLookup, lggr logger.Logger) { - q := url.Values{ - sl.FeedParamKey: {sl.Feeds[index]}, - sl.TimeParamKey: {sl.Time.String()}, - } - mercuryURL := r.mercury.cred.LegacyURL - reqUrl := fmt.Sprintf("%s%s%s", mercuryURL, mercuryPathV02, q.Encode()) - lggr.Debugf("request URL for upkeep %s feed %s: %s", sl.upkeepId.String(), sl.Feeds[index], reqUrl) - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqUrl, nil) - if err != nil { - ch <- MercuryData{Index: index, Error: err, Retryable: false, State: encoding.InvalidMercuryRequest} - return - } - - ts := time.Now().UTC().UnixMilli() - signature := r.generateHMAC(http.MethodGet, mercuryPathV02+q.Encode(), []byte{}, r.mercury.cred.Username, r.mercury.cred.Password, ts) - req.Header.Set(headerContentType, applicationJson) - req.Header.Set(headerAuthorization, r.mercury.cred.Username) - req.Header.Set(headerTimestamp, strconv.FormatInt(ts, 10)) - req.Header.Set(headerSignature, signature) - - // in the case of multiple retries here, use the last attempt's data - state := encoding.NoPipelineError - retryable := false - sent := false - retryErr := retry.Do( - func() error { - retryable = false - resp, err1 := r.hc.Do(req) - if err1 != nil { - lggr.Warnf("at block %s upkeep %s GET request fails for feed %s: %v", sl.Time.String(), sl.upkeepId.String(), sl.Feeds[index], err1) - retryable = true - state = encoding.MercuryFlakyFailure - return err1 - } - defer func(Body io.ReadCloser) { - err = Body.Close() - if err != nil { - lggr.Warnf("failed to close mercury response Body: %s", err) - } - }(resp.Body) - - body, err1 := io.ReadAll(resp.Body) - if err1 != nil { - retryable = false - state = encoding.InvalidMercuryResponse - return err1 - } - - if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusInternalServerError || resp.StatusCode == http.StatusBadGateway || resp.StatusCode == http.StatusServiceUnavailable || resp.StatusCode == http.StatusGatewayTimeout { - lggr.Warnf("at block %s upkeep %s received status code %d for feed %s", sl.Time.String(), sl.upkeepId.String(), resp.StatusCode, sl.Feeds[index]) - retryable = true - state = encoding.MercuryFlakyFailure - return errors.New(strconv.FormatInt(int64(resp.StatusCode), 10)) - } else if resp.StatusCode != http.StatusOK { - retryable = false - state = encoding.InvalidMercuryRequest - return fmt.Errorf("at block %s upkeep %s received status code %d for feed %s", sl.Time.String(), sl.upkeepId.String(), resp.StatusCode, sl.Feeds[index]) - } - - lggr.Debugf("at block %s upkeep %s received status code %d from mercury v0.2 with BODY=%s", sl.Time.String(), sl.upkeepId.String(), resp.StatusCode, hexutil.Encode(body)) - - var m MercuryV02Response - err1 = json.Unmarshal(body, &m) - if err1 != nil { - lggr.Warnf("at block %s upkeep %s failed to unmarshal body to MercuryV02Response for feed %s: %v", sl.Time.String(), sl.upkeepId.String(), sl.Feeds[index], err1) - retryable = false - state = encoding.MercuryUnmarshalError - return err1 - } - blobBytes, err1 := hexutil.Decode(m.ChainlinkBlob) - if err1 != nil { - lggr.Warnf("at block %s upkeep %s failed to decode chainlinkBlob %s for feed %s: %v", sl.Time.String(), sl.upkeepId.String(), m.ChainlinkBlob, sl.Feeds[index], err1) - retryable = false - state = encoding.InvalidMercuryResponse - return err1 - } - ch <- MercuryData{ - Index: index, - Bytes: [][]byte{blobBytes}, - Retryable: false, - State: encoding.NoPipelineError, - } - sent = true - return nil - }, - // only retry when the error is 404 Not Found, 500 Internal Server Error, 502 Bad Gateway, 503 Service Unavailable, 504 Gateway Timeout - retry.RetryIf(func(err error) bool { - return err.Error() == fmt.Sprintf("%d", http.StatusNotFound) || err.Error() == fmt.Sprintf("%d", http.StatusInternalServerError) || err.Error() == fmt.Sprintf("%d", http.StatusBadGateway) || err.Error() == fmt.Sprintf("%d", http.StatusServiceUnavailable) || err.Error() == fmt.Sprintf("%d", http.StatusGatewayTimeout) - }), - retry.Context(ctx), - retry.Delay(retryDelay), - retry.Attempts(totalAttempt)) - - if !sent { - md := MercuryData{ - Index: index, - Bytes: [][]byte{}, - Retryable: retryable, - Error: fmt.Errorf("failed to request feed for %s: %w", sl.Feeds[index], retryErr), - State: state, - } - ch <- md - } -} - -// multiFeedsRequest sends a Mercury v0.3 request for a multi-feed report -func (r *EvmRegistry) multiFeedsRequest(ctx context.Context, ch chan<- MercuryData, sl *StreamsLookup, lggr logger.Logger) { - // this won't work bc q.Encode() will encode commas as '%2C' but the server is strictly expecting a comma separated list - //q := url.Values{ - // feedIDs: {strings.Join(sl.Feeds, ",")}, - // timestamp: {sl.Time.String()}, - //} - params := fmt.Sprintf("%s=%s&%s=%s", feedIDs, strings.Join(sl.Feeds, ","), sl.TimeParamKey, sl.Time.String()) - batchPathV03 := mercuryBatchPathV03 - if sl.TimeParamKey == blockNumber { - batchPathV03 = mercuryBatchPathV03BlockNumber - } - reqUrl := fmt.Sprintf("%s%s%s", r.mercury.cred.URL, batchPathV03, params) - lggr.Debugf("request URL for upkeep %s userId %s: %s", sl.upkeepId.String(), r.mercury.cred.Username, reqUrl) - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqUrl, nil) - if err != nil { - ch <- MercuryData{Index: 0, Error: err, Retryable: false, State: encoding.InvalidMercuryRequest} - return - } - - ts := time.Now().UTC().UnixMilli() - signature := r.generateHMAC(http.MethodGet, batchPathV03+params, []byte{}, r.mercury.cred.Username, r.mercury.cred.Password, ts) - req.Header.Set(headerContentType, applicationJson) - // username here is often referred to as user id - req.Header.Set(headerAuthorization, r.mercury.cred.Username) - req.Header.Set(headerTimestamp, strconv.FormatInt(ts, 10)) - req.Header.Set(headerSignature, signature) - // mercury will inspect authorization headers above to make sure this user (in automation's context, this node) is eligible to access mercury - // and if it has an automation role. it will then look at this upkeep id to check if it has access to all the requested feeds. - req.Header.Set(headerUpkeepId, sl.upkeepId.String()) - - // in the case of multiple retries here, use the last attempt's data - state := encoding.NoPipelineError - retryable := false - sent := false - retryErr := retry.Do( - func() error { - retryable = false - resp, err1 := r.hc.Do(req) - if err1 != nil { - lggr.Warnf("at timestamp %s upkeep %s GET request fails from mercury v0.3: %v", sl.Time.String(), sl.upkeepId.String(), err1) - retryable = true - state = encoding.MercuryFlakyFailure - return err1 - } - defer func(Body io.ReadCloser) { - err = Body.Close() - if err != nil { - lggr.Warnf("failed to close mercury response Body: %s", err) - } - }(resp.Body) - - body, err1 := io.ReadAll(resp.Body) - if err1 != nil { - retryable = false - state = encoding.InvalidMercuryResponse - return err1 - } - - lggr.Infof("at timestamp %s upkeep %s received status code %d from mercury v0.3", sl.Time.String(), sl.upkeepId.String(), resp.StatusCode) - if resp.StatusCode == http.StatusUnauthorized { - retryable = false - state = encoding.UpkeepNotAuthorized - return fmt.Errorf("at timestamp %s upkeep %s received status code %d from mercury v0.3, most likely this is caused by unauthorized upkeep", sl.Time.String(), sl.upkeepId.String(), resp.StatusCode) - } else if resp.StatusCode == http.StatusBadRequest { - retryable = false - state = encoding.InvalidMercuryRequest - return fmt.Errorf("at timestamp %s upkeep %s received status code %d from mercury v0.3 with message: %s", sl.Time.String(), sl.upkeepId.String(), resp.StatusCode, string(body)) - } else if resp.StatusCode == http.StatusInternalServerError || resp.StatusCode == http.StatusBadGateway || resp.StatusCode == http.StatusServiceUnavailable || resp.StatusCode == http.StatusGatewayTimeout { - retryable = true - state = encoding.MercuryFlakyFailure - return fmt.Errorf("%d", resp.StatusCode) - } else if resp.StatusCode == http.StatusPartialContent { - // TODO (AUTO-5044): handle response code 206 entirely with errors field parsing - lggr.Warnf("at timestamp %s upkeep %s requested [%s] feeds but mercury v0.3 server returned 206 status, treating it as 404 and retrying", sl.Time.String(), sl.upkeepId.String(), sl.Feeds) - retryable = true - state = encoding.MercuryFlakyFailure - return fmt.Errorf("%d", http.StatusPartialContent) - } else if resp.StatusCode != http.StatusOK { - retryable = false - state = encoding.InvalidMercuryRequest - return fmt.Errorf("at timestamp %s upkeep %s received status code %d from mercury v0.3", sl.Time.String(), sl.upkeepId.String(), resp.StatusCode) - } - - lggr.Debugf("at block %s upkeep %s received status code %d from mercury v0.3 with BODY=%s", sl.Time.String(), sl.upkeepId.String(), resp.StatusCode, hexutil.Encode(body)) - - var response MercuryV03Response - err1 = json.Unmarshal(body, &response) - if err1 != nil { - lggr.Warnf("at timestamp %s upkeep %s failed to unmarshal body to MercuryV03Response from mercury v0.3: %v", sl.Time.String(), sl.upkeepId.String(), err1) - retryable = false - state = encoding.MercuryUnmarshalError - return err1 - } - // in v0.3, if some feeds are not available, the server will only return available feeds, but we need to make sure ALL feeds are retrieved before calling user contract - // hence, retry in this case. retry will help when we send a very new timestamp and reports are not yet generated - if len(response.Reports) != len(sl.Feeds) { - var receivedFeeds []string - for _, f := range response.Reports { - receivedFeeds = append(receivedFeeds, f.FeedID) - } - lggr.Warnf("at timestamp %s upkeep %s mercury v0.3 server returned 206 status with [%s] reports while we requested [%s] feeds, retrying", sl.Time.String(), sl.upkeepId.String(), receivedFeeds, sl.Feeds) - retryable = true - state = encoding.MercuryFlakyFailure - return fmt.Errorf("%d", http.StatusNotFound) - } - var reportBytes [][]byte - for _, rsp := range response.Reports { - b, err := hexutil.Decode(rsp.FullReport) - if err != nil { - lggr.Warnf("at timestamp %s upkeep %s failed to decode reportBlob %s: %v", sl.Time.String(), sl.upkeepId.String(), rsp.FullReport, err) - retryable = false - state = encoding.InvalidMercuryResponse - return err - } - reportBytes = append(reportBytes, b) - } - ch <- MercuryData{ - Index: 0, - Bytes: reportBytes, - Retryable: false, - State: encoding.NoPipelineError, - } - sent = true - return nil - }, - // only retry when the error is 206 Partial Content, 404 Not Found, 500 Internal Server Error, 502 Bad Gateway, 503 Service Unavailable, 504 Gateway Timeout - retry.RetryIf(func(err error) bool { - return err.Error() == fmt.Sprintf("%d", http.StatusPartialContent) || err.Error() == fmt.Sprintf("%d", http.StatusNotFound) || err.Error() == fmt.Sprintf("%d", http.StatusInternalServerError) || err.Error() == fmt.Sprintf("%d", http.StatusBadGateway) || err.Error() == fmt.Sprintf("%d", http.StatusServiceUnavailable) || err.Error() == fmt.Sprintf("%d", http.StatusGatewayTimeout) - }), - retry.Context(ctx), - retry.Delay(retryDelay), - retry.Attempts(totalAttempt)) - - if !sent { - md := MercuryData{ - Index: 0, - Bytes: [][]byte{}, - Retryable: retryable, - Error: retryErr, - State: state, - } - ch <- md - } -} - -// generateHMAC calculates a user HMAC for Mercury server authentication. -func (r *EvmRegistry) generateHMAC(method string, path string, body []byte, clientId string, secret string, ts int64) string { - bodyHash := sha256.New() - bodyHash.Write(body) - hashString := fmt.Sprintf("%s %s %s %s %d", - method, - path, - hex.EncodeToString(bodyHash.Sum(nil)), - clientId, - ts) - signedMessage := hmac.New(sha256.New, []byte(secret)) - signedMessage.Write([]byte(hashString)) - userHmac := hex.EncodeToString(signedMessage.Sum(nil)) - return userHmac -} - -// calculateRetryConfig returns plugin retry interval based on how many times plugin has retried this work -func (r *EvmRegistry) calculateRetryConfig(prk string) time.Duration { - var ri time.Duration - var retries int - totalAttempts, ok := r.mercury.pluginRetryCache.Get(prk) - if ok { - retries = totalAttempts.(int) - if retries < totalFastPluginRetries { - ri = 1 * time.Second - } else if retries < totalMediumPluginRetries { - ri = 5 * time.Second - } - // if the core node has retried totalMediumPluginRetries times, do not set retry interval and plugin will use - // the default interval - } else { - ri = 1 * time.Second - } - r.mercury.pluginRetryCache.Set(prk, retries+1, cache.DefaultExpiration) - return ri -} - -// generatePluginRetryKey returns a plugin retry cache key -func generatePluginRetryKey(workID string, block uint64) string { - return workID + "|" + fmt.Sprintf("%d", block) -} diff --git a/core/services/ocr2/plugins/ocr2keeper/evm21/streams_lookup_test.go b/core/services/ocr2/plugins/ocr2keeper/evm21/streams_lookup_test.go deleted file mode 100644 index 8d7c67d80ce..00000000000 --- a/core/services/ocr2/plugins/ocr2keeper/evm21/streams_lookup_test.go +++ /dev/null @@ -1,1341 +0,0 @@ -package evm - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "math/big" - "net/http" - "strings" - "testing" - "time" - - "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/patrickmn/go-cache" - "github.com/pkg/errors" - ocr2keepers "github.com/smartcontractkit/ocr2keepers/pkg/v3/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/encoding" - "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mocks" - - evmClientMocks "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client/mocks" - "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/i_keeper_registry_master_wrapper_2_1" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/streams_lookup_compatible_interface" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/models" -) - -// setups up an evm registry for tests. -func setupEVMRegistry(t *testing.T) *EvmRegistry { - lggr := logger.TestLogger(t) - addr := common.HexToAddress("0x6cA639822c6C241Fa9A7A6b5032F6F7F1C513CAD") - keeperRegistryABI, err := abi.JSON(strings.NewReader(i_keeper_registry_master_wrapper_2_1.IKeeperRegistryMasterABI)) - require.Nil(t, err, "need registry abi") - streamsLookupCompatibleABI, err := abi.JSON(strings.NewReader(streams_lookup_compatible_interface.StreamsLookupCompatibleInterfaceABI)) - require.Nil(t, err, "need mercury abi") - var logPoller logpoller.LogPoller - mockReg := mocks.NewRegistry(t) - mockHttpClient := mocks.NewHttpClient(t) - client := evmClientMocks.NewClient(t) - - r := &EvmRegistry{ - lggr: lggr, - poller: logPoller, - addr: addr, - client: client, - logProcessed: make(map[string]bool), - registry: mockReg, - abi: keeperRegistryABI, - active: NewActiveUpkeepList(), - packer: encoding.NewAbiPacker(), - headFunc: func(ocr2keepers.BlockKey) {}, - chLog: make(chan logpoller.Log, 1000), - mercury: &MercuryConfig{ - cred: &models.MercuryCredentials{ - LegacyURL: "https://google.old.com", - URL: "https://google.com", - Username: "FakeClientID", - Password: "FakeClientKey", - }, - abi: streamsLookupCompatibleABI, - allowListCache: cache.New(defaultAllowListExpiration, cleanupInterval), - pluginRetryCache: cache.New(defaultPluginRetryExpiration, cleanupInterval), - }, - hc: mockHttpClient, - } - return r -} - -func TestEvmRegistry_StreamsLookup(t *testing.T) { - upkeepId, ok := new(big.Int).SetString("71022726777042968814359024671382968091267501884371696415772139504780367423725", 10) - var upkeepIdentifier [32]byte - copy(upkeepIdentifier[:], upkeepId.Bytes()) - assert.True(t, ok, t.Name()) - blockNum := ocr2keepers.BlockNumber(37974374) - tests := []struct { - name string - input []ocr2keepers.CheckResult - blobs map[string]string - callbackResp []byte - expectedResults []ocr2keepers.CheckResult - callbackNeeded bool - extraData []byte - checkCallbackResp []byte - values [][]byte - cachedAdminCfg bool - hasError bool - hasPermission bool - v3 bool - }{ - { - name: "success - happy path no cache", - input: []ocr2keepers.CheckResult{ - { - PerformData: hexutil.MustDecode("0xf055e4a200000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000e0000000000000000000000000000000000000000000000000000000000000024000000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000000000280000000000000000000000000000000000000000000000000000000000000000966656564496448657800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000423078343535343438326435353533343432643431353234323439353435323535346432643534343535333534346534353534303030303030303030303030303030300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000042307834323534343332643535353334343264343135323432343935343532353534643264353434353533353434653435353430303030303030303030303030303030000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000b626c6f636b4e756d62657200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000064000000000000000000000000"), - UpkeepID: upkeepIdentifier, - Trigger: ocr2keepers.Trigger{ - BlockNumber: blockNum, - }, - IneligibilityReason: uint8(encoding.UpkeepFailureReasonTargetCheckReverted), - }, - }, - blobs: map[string]string{ - "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000": "0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa3", - "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000": "0x0006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d", - }, - cachedAdminCfg: false, - extraData: hexutil.MustDecode("0x0000000000000000000000000000000000000064"), - callbackNeeded: true, - checkCallbackResp: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000063a400000000000000000000000000000000000000000000000000000000000006e0000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000006a000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000034000000000000000000000000000000000000000000000000000000000000002e000066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa300000000000000000000000000000000000000000000000000000000000002e00006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d00000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000064000000000000000000000000"), - values: [][]byte{hexutil.MustDecode("0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa3"), hexutil.MustDecode("0x0006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d")}, - expectedResults: []ocr2keepers.CheckResult{ - { - Eligible: true, - PerformData: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000006a000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000034000000000000000000000000000000000000000000000000000000000000002e000066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa300000000000000000000000000000000000000000000000000000000000002e00006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d00000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000064000000000000000000000000"), - UpkeepID: upkeepIdentifier, - Trigger: ocr2keepers.Trigger{ - BlockNumber: blockNum, - }, - IneligibilityReason: uint8(encoding.UpkeepFailureReasonNone), - }, - }, - hasPermission: true, - }, - { - name: "success - happy path no cache - v0.3", - input: []ocr2keepers.CheckResult{ - { - PerformData: hexutil.MustDecode("0xf055e4a200000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000e0000000000000000000000000000000000000000000000000000000000000024000000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000000000280000000000000000000000000000000000000000000000000000000000000000766656564494473000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000423078343535343438326435353533343432643431353234323439353435323535346432643534343535333534346534353534303030303030303030303030303030300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000042307834323534343332643535353334343264343135323432343935343532353534643264353434353533353434653435353430303030303030303030303030303030000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000b626c6f636b4e756d62657200000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000064000000000000000000000000"), - UpkeepID: upkeepIdentifier, - Trigger: ocr2keepers.Trigger{ - BlockNumber: blockNum, - }, - IneligibilityReason: uint8(encoding.UpkeepFailureReasonTargetCheckReverted), - }, - }, - blobs: map[string]string{ - "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000": "0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa3", - "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000": "0x0006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d", - }, - cachedAdminCfg: false, - extraData: hexutil.MustDecode("0x0000000000000000000000000000000000000064"), - callbackNeeded: true, - checkCallbackResp: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000063a400000000000000000000000000000000000000000000000000000000000006e0000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000006a000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000034000000000000000000000000000000000000000000000000000000000000002e000066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa300000000000000000000000000000000000000000000000000000000000002e00006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d00000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000064000000000000000000000000"), - values: [][]byte{hexutil.MustDecode("0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa3"), hexutil.MustDecode("0x0006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d")}, - expectedResults: []ocr2keepers.CheckResult{ - { - Eligible: true, - PerformData: hexutil.MustDecode("0x000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000006a000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000034000000000000000000000000000000000000000000000000000000000000002e000066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000004555638000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280010100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000000269ecbb83b000000000000000000000000000000000000000000000000000000269e4a4e14000000000000000000000000000000000000000000000000000000269f4d0edb000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002381e91cffa9502c20de1ddcee350db3f715a5ab449448e3184a5b03c682356c6e2115f20663b3731e373cf33465a96da26f2876debb548f281e62e48f64c374200000000000000000000000000000000000000000000000000000000000000027db99e34135098d4e0bb9ae143ec9cd72fd63150c6d0cc5b38f4aa1aa42408377e8fe8e5ac489c9b7f62ff5aa7b05d2e892e7dee4cac631097247969b3b03fa300000000000000000000000000000000000000000000000000000000000002e00006da4a86c4933dd4a87b21dd2871aea29f706bcde43c70039355ac5b664fb5000000000000000000000000000000000000000000000000000000000454d118000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204254432d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064f0d4a0000000000000000000000000000000000000000000000000000002645f00877a000000000000000000000000000000000000000000000000000002645e1e1010000000000000000000000000000000000000000000000000000002645fe2fee4000000000000000000000000000000000000000000000000000000000243716664b42d20423a47fb13ad3098b49b37f667548e6745fff958b663afe25a845f6100000000000000000000000000000000000000000000000000000000024371660000000000000000000000000000000000000000000000000000000064f0d4a00000000000000000000000000000000000000000000000000000000000000002a0373c0bce7393673f819eb9681cac2773c2d718ce933eb858252195b17a9c832d7b0bee173c02c3c25fb65912b8b13b9302ede8423bab3544cb7a8928d5eb3600000000000000000000000000000000000000000000000000000000000000027d7b79d7646383a5dbf51edf14d53bd3ad0a9f3ca8affab3165e89d3ddce9cb17b58e892fafe4ecb24d2fde07c6a756029e752a5114c33c173df4e7d309adb4d00000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000064000000000000000000000000"), - UpkeepID: upkeepIdentifier, - Trigger: ocr2keepers.Trigger{ - BlockNumber: blockNum, - }, - IneligibilityReason: uint8(encoding.UpkeepFailureReasonNone), - }, - }, - hasPermission: true, - v3: true, - }, - { - name: "skip - failure reason is insufficient balance", - input: []ocr2keepers.CheckResult{ - { - PerformData: []byte{}, - UpkeepID: upkeepIdentifier, - Trigger: ocr2keepers.Trigger{ - BlockNumber: 26046145, - }, - IneligibilityReason: uint8(encoding.UpkeepFailureReasonInsufficientBalance), - }, - }, - expectedResults: []ocr2keepers.CheckResult{ - { - Eligible: false, - PerformData: []byte{}, - UpkeepID: upkeepIdentifier, - Trigger: ocr2keepers.Trigger{ - BlockNumber: 26046145, - }, - IneligibilityReason: uint8(encoding.UpkeepFailureReasonInsufficientBalance), - }, - }, - hasError: true, - }, - { - name: "skip - invalid revert data", - input: []ocr2keepers.CheckResult{ - { - PerformData: []byte{}, - UpkeepID: upkeepIdentifier, - Trigger: ocr2keepers.Trigger{ - BlockNumber: 26046145, - }, - IneligibilityReason: uint8(encoding.UpkeepFailureReasonTargetCheckReverted), - }, - }, - expectedResults: []ocr2keepers.CheckResult{ - { - Eligible: false, - PerformData: []byte{}, - UpkeepID: upkeepIdentifier, - Trigger: ocr2keepers.Trigger{ - BlockNumber: 26046145, - }, - IneligibilityReason: uint8(encoding.UpkeepFailureReasonTargetCheckReverted), - }, - }, - hasError: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := setupEVMRegistry(t) - client := new(evmClientMocks.Client) - r.client = client - - if !tt.cachedAdminCfg && !tt.hasError { - cfg := UpkeepPrivilegeConfig{MercuryEnabled: tt.hasPermission} - bCfg, err := json.Marshal(cfg) - require.Nil(t, err) - - bContractCfg, err := r.abi.Methods["getUpkeepPrivilegeConfig"].Outputs.PackValues([]interface{}{bCfg}) - require.Nil(t, err) - - payload, err := r.abi.Pack("getUpkeepPrivilegeConfig", upkeepId) - require.Nil(t, err) - - args := map[string]interface{}{ - "to": r.addr.Hex(), - "data": hexutil.Bytes(payload), - } - - client.On("CallContext", mock.Anything, mock.AnythingOfType("*hexutil.Bytes"), "eth_call", args, mock.AnythingOfType("string")).Return(nil). - Run(func(args mock.Arguments) { - b := args.Get(1).(*hexutil.Bytes) - *b = bContractCfg - }).Once() - } - - if len(tt.blobs) > 0 { - if tt.v3 { - hc := mocks.NewHttpClient(t) - mr1 := MercuryV03Response{ - Reports: []MercuryV03Report{{FullReport: tt.blobs["0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"]}, {FullReport: tt.blobs["0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"]}}} - b1, err := json.Marshal(mr1) - assert.Nil(t, err) - resp1 := &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewReader(b1)), - } - hc.On("Do", mock.Anything).Return(resp1, nil).Once() - r.hc = hc - } else { - hc := mocks.NewHttpClient(t) - mr1 := MercuryV02Response{ChainlinkBlob: tt.blobs["0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"]} - b1, err := json.Marshal(mr1) - assert.Nil(t, err) - resp1 := &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewReader(b1)), - } - mr2 := MercuryV02Response{ChainlinkBlob: tt.blobs["0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"]} - b2, err := json.Marshal(mr2) - assert.Nil(t, err) - resp2 := &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewReader(b2)), - } - hc.On("Do", mock.MatchedBy(func(req *http.Request) bool { - return strings.Contains(req.URL.String(), "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000") - })).Return(resp2, nil).Once() - - hc.On("Do", mock.MatchedBy(func(req *http.Request) bool { - return strings.Contains(req.URL.String(), "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000") - })).Return(resp1, nil).Once() - r.hc = hc - } - } - - if tt.callbackNeeded { - payload, err := r.abi.Pack("checkCallback", upkeepId, tt.values, tt.extraData) - require.Nil(t, err) - args := map[string]interface{}{ - "to": r.addr.Hex(), - "data": hexutil.Bytes(payload), - } - client.On("CallContext", mock.Anything, mock.AnythingOfType("*hexutil.Bytes"), "eth_call", args, hexutil.EncodeUint64(uint64(blockNum))).Return(nil). - Run(func(args mock.Arguments) { - b := args.Get(1).(*hexutil.Bytes) - *b = tt.checkCallbackResp - }).Once() - } - - got := r.streamsLookup(context.Background(), tt.input) - assert.Equal(t, tt.expectedResults, got, tt.name) - }) - } -} - -func TestEvmRegistry_AllowedToUseMercury(t *testing.T) { - upkeepId, ok := new(big.Int).SetString("71022726777042968814359024671382968091267501884371696415772139504780367423725", 10) - assert.True(t, ok, t.Name()) - tests := []struct { - name string - cached bool - allowed bool - ethCallErr error - err error - state encoding.PipelineExecutionState - reason encoding.UpkeepFailureReason - retryable bool - config []byte - }{ - { - name: "success - allowed via cache", - cached: true, - allowed: true, - }, - { - name: "success - allowed via fetching privilege config", - allowed: true, - }, - { - name: "success - not allowed via cache", - cached: true, - allowed: false, - }, - { - name: "success - not allowed via fetching privilege config", - allowed: false, - }, - { - name: "failure - cannot unmarshal privilege config", - err: fmt.Errorf("failed to unmarshal privilege config: invalid character '\\x00' looking for beginning of value"), - state: encoding.MercuryUnmarshalError, - config: []byte{0, 1}, - }, - { - name: "failure - flaky RPC", - retryable: true, - err: fmt.Errorf("failed to get upkeep privilege config: flaky RPC"), - state: encoding.RpcFlakyFailure, - ethCallErr: fmt.Errorf("flaky RPC"), - }, - { - name: "failure - empty upkeep privilege config", - err: fmt.Errorf("upkeep privilege config is empty"), - reason: encoding.UpkeepFailureReasonMercuryAccessNotAllowed, - config: []byte{}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := setupEVMRegistry(t) - - client := new(evmClientMocks.Client) - r.client = client - - if tt.cached { - r.mercury.allowListCache.Set(upkeepId.String(), tt.allowed, cache.DefaultExpiration) - } else { - if tt.err != nil { - bContractCfg, err := r.abi.Methods["getUpkeepPrivilegeConfig"].Outputs.PackValues([]interface{}{tt.config}) - require.Nil(t, err) - - payload, err := r.abi.Pack("getUpkeepPrivilegeConfig", upkeepId) - require.Nil(t, err) - - args := map[string]interface{}{ - "to": r.addr.Hex(), - "data": hexutil.Bytes(payload), - } - - client.On("CallContext", mock.Anything, mock.AnythingOfType("*hexutil.Bytes"), "eth_call", args, mock.AnythingOfType("string")). - Return(tt.ethCallErr). - Run(func(args mock.Arguments) { - b := args.Get(1).(*hexutil.Bytes) - *b = bContractCfg - }).Once() - } else { - cfg := UpkeepPrivilegeConfig{MercuryEnabled: tt.allowed} - bCfg, err := json.Marshal(cfg) - require.Nil(t, err) - - bContractCfg, err := r.abi.Methods["getUpkeepPrivilegeConfig"].Outputs.PackValues([]interface{}{bCfg}) - require.Nil(t, err) - - payload, err := r.abi.Pack("getUpkeepPrivilegeConfig", upkeepId) - require.Nil(t, err) - - args := map[string]interface{}{ - "to": r.addr.Hex(), - "data": hexutil.Bytes(payload), - } - - client.On("CallContext", mock.Anything, mock.AnythingOfType("*hexutil.Bytes"), "eth_call", args, mock.AnythingOfType("string")).Return(nil). - Run(func(args mock.Arguments) { - b := args.Get(1).(*hexutil.Bytes) - *b = bContractCfg - }).Once() - } - } - - opts := &bind.CallOpts{ - BlockNumber: big.NewInt(10), - } - - state, reason, retryable, allowed, err := r.allowedToUseMercury(opts, upkeepId) - assert.Equal(t, tt.err, err) - assert.Equal(t, tt.allowed, allowed) - assert.Equal(t, tt.state, state) - assert.Equal(t, tt.reason, reason) - assert.Equal(t, tt.retryable, retryable) - }) - } -} - -func TestEvmRegistry_DoMercuryRequestV02(t *testing.T) { - upkeepId, _ := new(big.Int).SetString("88786950015966611018675766524283132478093844178961698330929478019253453382042", 10) - - tests := []struct { - name string - lookup *StreamsLookup - mockHttpStatusCode int - mockChainlinkBlobs []string - pluginRetries int - pluginRetryKey string - expectedValues [][]byte - expectedRetryable bool - expectedRetryInterval time.Duration - expectedError error - state encoding.PipelineExecutionState - reason encoding.UpkeepFailureReason - }{ - { - name: "success", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIdHex, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: blockNumber, - Time: big.NewInt(25880526), - ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, - }, - upkeepId: upkeepId, - }, - mockHttpStatusCode: http.StatusOK, - mockChainlinkBlobs: []string{"0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000000081401000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000289ad8d367000000000000000000000000000000000000000000000000000000289acf0b38000000000000000000000000000000000000000000000000000000289b3da40000000000000000000000000000000000000000000000000000000000018ae7ce74d9fa252a8983976eab600dc7590c778d04813430841bc6e765c34cd81a168d00000000000000000000000000000000000000000000000000000000018ae7cb0000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000000000000260412b94e525ca6cedc9f544fd86f77606d52fe731a5d069dbe836a8bfc0fb8c911963b0ae7a14971f3b4621bffb802ef0605392b9a6c89c7fab1df8633a5ade00000000000000000000000000000000000000000000000000000000000000024500c2f521f83fba5efc2bf3effaaedde43d0a4adff785c1213b712a3aed0d8157642a84324db0cf9695ebd27708d4608eb0337e0dd87b0e43f0fa70c700d911"}, - expectedValues: [][]byte{{0, 6, 109, 252, 209, 237, 45, 149, 177, 140, 148, 141, 188, 91, 214, 76, 104, 122, 254, 147, 228, 202, 125, 102, 61, 222, 193, 76, 32, 9, 10, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 128, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 69, 84, 72, 45, 85, 83, 68, 45, 65, 82, 66, 73, 84, 82, 85, 77, 45, 84, 69, 83, 84, 78, 69, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 137, 28, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 154, 216, 211, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 154, 207, 11, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 155, 61, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 138, 231, 206, 116, 217, 250, 37, 42, 137, 131, 151, 110, 171, 96, 13, 199, 89, 12, 119, 141, 4, 129, 52, 48, 132, 27, 198, 231, 101, 195, 76, 216, 26, 22, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 138, 231, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 137, 28, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 96, 65, 43, 148, 229, 37, 202, 108, 237, 201, 245, 68, 253, 134, 247, 118, 6, 213, 47, 231, 49, 165, 208, 105, 219, 232, 54, 168, 191, 192, 251, 140, 145, 25, 99, 176, 174, 122, 20, 151, 31, 59, 70, 33, 191, 251, 128, 46, 240, 96, 83, 146, 185, 166, 200, 156, 127, 171, 29, 248, 99, 58, 90, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 69, 0, 194, 245, 33, 248, 63, 186, 94, 252, 43, 243, 239, 250, 174, 221, 228, 61, 10, 74, 223, 247, 133, 193, 33, 59, 113, 42, 58, 237, 13, 129, 87, 100, 42, 132, 50, 77, 176, 207, 150, 149, 235, 210, 119, 8, 212, 96, 142, 176, 51, 126, 13, 216, 123, 14, 67, 240, 250, 112, 199, 0, 217, 17}}, - expectedRetryable: false, - expectedError: nil, - }, - { - name: "failure - retryable and interval is 1s", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIdHex, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: blockNumber, - Time: big.NewInt(25880526), - ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, - }, - upkeepId: upkeepId, - }, - mockHttpStatusCode: http.StatusInternalServerError, - mockChainlinkBlobs: []string{"0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000000081401000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000289ad8d367000000000000000000000000000000000000000000000000000000289acf0b38000000000000000000000000000000000000000000000000000000289b3da40000000000000000000000000000000000000000000000000000000000018ae7ce74d9fa252a8983976eab600dc7590c778d04813430841bc6e765c34cd81a168d00000000000000000000000000000000000000000000000000000000018ae7cb0000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000000000000260412b94e525ca6cedc9f544fd86f77606d52fe731a5d069dbe836a8bfc0fb8c911963b0ae7a14971f3b4621bffb802ef0605392b9a6c89c7fab1df8633a5ade00000000000000000000000000000000000000000000000000000000000000024500c2f521f83fba5efc2bf3effaaedde43d0a4adff785c1213b712a3aed0d8157642a84324db0cf9695ebd27708d4608eb0337e0dd87b0e43f0fa70c700d911"}, - expectedValues: [][]byte{nil}, - expectedRetryable: true, - pluginRetries: 0, - expectedRetryInterval: 1 * time.Second, - expectedError: errors.New("failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000: All attempts fail:\n#1: 500\n#2: 500\n#3: 500"), - state: encoding.MercuryFlakyFailure, - }, - { - name: "failure - retryable and interval is 5s", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIdHex, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: blockNumber, - Time: big.NewInt(25880526), - ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, - }, - upkeepId: upkeepId, - }, - pluginRetries: 5, - mockHttpStatusCode: http.StatusInternalServerError, - mockChainlinkBlobs: []string{"0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000000081401000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000289ad8d367000000000000000000000000000000000000000000000000000000289acf0b38000000000000000000000000000000000000000000000000000000289b3da40000000000000000000000000000000000000000000000000000000000018ae7ce74d9fa252a8983976eab600dc7590c778d04813430841bc6e765c34cd81a168d00000000000000000000000000000000000000000000000000000000018ae7cb0000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000000000000260412b94e525ca6cedc9f544fd86f77606d52fe731a5d069dbe836a8bfc0fb8c911963b0ae7a14971f3b4621bffb802ef0605392b9a6c89c7fab1df8633a5ade00000000000000000000000000000000000000000000000000000000000000024500c2f521f83fba5efc2bf3effaaedde43d0a4adff785c1213b712a3aed0d8157642a84324db0cf9695ebd27708d4608eb0337e0dd87b0e43f0fa70c700d911"}, - expectedValues: [][]byte{nil}, - expectedRetryable: true, - expectedRetryInterval: 5 * time.Second, - expectedError: errors.New("failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000: All attempts fail:\n#1: 500\n#2: 500\n#3: 500"), - state: encoding.MercuryFlakyFailure, - }, - { - name: "failure - not retryable because there are many plugin retries already", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIdHex, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: blockNumber, - Time: big.NewInt(25880526), - ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, - }, - upkeepId: upkeepId, - }, - pluginRetries: 10, - mockHttpStatusCode: http.StatusInternalServerError, - mockChainlinkBlobs: []string{"0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000000081401000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000289ad8d367000000000000000000000000000000000000000000000000000000289acf0b38000000000000000000000000000000000000000000000000000000289b3da40000000000000000000000000000000000000000000000000000000000018ae7ce74d9fa252a8983976eab600dc7590c778d04813430841bc6e765c34cd81a168d00000000000000000000000000000000000000000000000000000000018ae7cb0000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000000000000260412b94e525ca6cedc9f544fd86f77606d52fe731a5d069dbe836a8bfc0fb8c911963b0ae7a14971f3b4621bffb802ef0605392b9a6c89c7fab1df8633a5ade00000000000000000000000000000000000000000000000000000000000000024500c2f521f83fba5efc2bf3effaaedde43d0a4adff785c1213b712a3aed0d8157642a84324db0cf9695ebd27708d4608eb0337e0dd87b0e43f0fa70c700d911"}, - expectedValues: [][]byte{nil}, - expectedRetryable: true, - expectedError: errors.New("failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000: All attempts fail:\n#1: 500\n#2: 500\n#3: 500"), - state: encoding.MercuryFlakyFailure, - }, - { - name: "failure - not retryable", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIdHex, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: blockNumber, - Time: big.NewInt(25880526), - ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, - }, - upkeepId: upkeepId, - }, - mockHttpStatusCode: http.StatusTooManyRequests, - mockChainlinkBlobs: []string{"0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000000081401000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000289ad8d367000000000000000000000000000000000000000000000000000000289acf0b38000000000000000000000000000000000000000000000000000000289b3da40000000000000000000000000000000000000000000000000000000000018ae7ce74d9fa252a8983976eab600dc7590c778d04813430841bc6e765c34cd81a168d00000000000000000000000000000000000000000000000000000000018ae7cb0000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000000000000260412b94e525ca6cedc9f544fd86f77606d52fe731a5d069dbe836a8bfc0fb8c911963b0ae7a14971f3b4621bffb802ef0605392b9a6c89c7fab1df8633a5ade00000000000000000000000000000000000000000000000000000000000000024500c2f521f83fba5efc2bf3effaaedde43d0a4adff785c1213b712a3aed0d8157642a84324db0cf9695ebd27708d4608eb0337e0dd87b0e43f0fa70c700d911"}, - expectedValues: [][]byte{nil}, - expectedRetryable: false, - expectedError: errors.New("failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000: All attempts fail:\n#1: at block 25880526 upkeep 88786950015966611018675766524283132478093844178961698330929478019253453382042 received status code 429 for feed 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"), - state: encoding.InvalidMercuryRequest, - }, - { - name: "failure - no feeds", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIdHex, - Feeds: []string{}, - TimeParamKey: blockNumber, - Time: big.NewInt(25880526), - ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, - }, - upkeepId: upkeepId, - }, - expectedValues: [][]byte{}, - reason: encoding.UpkeepFailureReasonInvalidRevertDataInput, - }, - { - name: "failure - invalid revert data", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIDs, - Feeds: []string{}, - TimeParamKey: blockNumber, - Time: big.NewInt(25880526), - ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, - }, - upkeepId: upkeepId, - }, - expectedValues: [][]byte{}, - reason: encoding.UpkeepFailureReasonInvalidRevertDataInput, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := setupEVMRegistry(t) - if tt.pluginRetries != 0 { - r.mercury.pluginRetryCache.Set(tt.pluginRetryKey, tt.pluginRetries, cache.DefaultExpiration) - } - hc := mocks.NewHttpClient(t) - - for _, blob := range tt.mockChainlinkBlobs { - mr := MercuryV02Response{ChainlinkBlob: blob} - b, err := json.Marshal(mr) - assert.Nil(t, err) - - resp := &http.Response{ - StatusCode: tt.mockHttpStatusCode, - Body: io.NopCloser(bytes.NewReader(b)), - } - if tt.expectedError != nil && tt.expectedRetryable || tt.pluginRetries > 0 { - hc.On("Do", mock.Anything).Return(resp, nil).Times(totalAttempt) - } else { - hc.On("Do", mock.Anything).Return(resp, nil).Once() - } - } - r.hc = hc - - state, reason, values, retryable, ri, reqErr := r.doMercuryRequest(context.Background(), tt.lookup, tt.pluginRetryKey, r.lggr) - assert.Equal(t, tt.expectedValues, values) - assert.Equal(t, tt.expectedRetryable, retryable) - if retryable { - newRetries, _ := r.mercury.pluginRetryCache.Get(tt.pluginRetryKey) - assert.Equal(t, tt.pluginRetries+1, newRetries.(int)) - } - assert.Equal(t, tt.expectedRetryInterval, ri) - assert.Equal(t, tt.state, state) - assert.Equal(t, tt.reason, reason) - if tt.expectedError != nil { - assert.True(t, strings.HasPrefix(reqErr.Error(), "failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000")) - } - }) - } -} - -func TestEvmRegistry_DoMercuryRequestV03(t *testing.T) { - upkeepId, _ := new(big.Int).SetString("88786950015966611018675766524283132478093844178961698330929478019253453382042", 10) - - tests := []struct { - name string - lookup *StreamsLookup - mockHttpStatusCode int - mockChainlinkBlobs []string - pluginRetryKey string - expectedValues [][]byte - expectedRetryable bool - expectedRetryInterval time.Duration - expectedError error - state encoding.PipelineExecutionState - reason encoding.UpkeepFailureReason - }{ - { - name: "success v0.3", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIDs, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: blockNumber, - Time: big.NewInt(25880526), - ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, - }, - upkeepId: upkeepId, - }, - mockHttpStatusCode: http.StatusOK, - mockChainlinkBlobs: []string{"0x00066dfcd1ed2d95b18c948dbc5bd64c687afe93e4ca7d663ddec14c20090ad80000000000000000000000000000000000000000000000000000000000081401000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000002200000000000000000000000000000000000000000000000000000000000000280000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001204554482d5553442d415242495452554d2d544553544e455400000000000000000000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000289ad8d367000000000000000000000000000000000000000000000000000000289acf0b38000000000000000000000000000000000000000000000000000000289b3da40000000000000000000000000000000000000000000000000000000000018ae7ce74d9fa252a8983976eab600dc7590c778d04813430841bc6e765c34cd81a168d00000000000000000000000000000000000000000000000000000000018ae7cb0000000000000000000000000000000000000000000000000000000064891c98000000000000000000000000000000000000000000000000000000000000000260412b94e525ca6cedc9f544fd86f77606d52fe731a5d069dbe836a8bfc0fb8c911963b0ae7a14971f3b4621bffb802ef0605392b9a6c89c7fab1df8633a5ade00000000000000000000000000000000000000000000000000000000000000024500c2f521f83fba5efc2bf3effaaedde43d0a4adff785c1213b712a3aed0d8157642a84324db0cf9695ebd27708d4608eb0337e0dd87b0e43f0fa70c700d911"}, - expectedValues: [][]byte{{0, 6, 109, 252, 209, 237, 45, 149, 177, 140, 148, 141, 188, 91, 214, 76, 104, 122, 254, 147, 228, 202, 125, 102, 61, 222, 193, 76, 32, 9, 10, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 128, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 32, 69, 84, 72, 45, 85, 83, 68, 45, 65, 82, 66, 73, 84, 82, 85, 77, 45, 84, 69, 83, 84, 78, 69, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 137, 28, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 154, 216, 211, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 154, 207, 11, 56, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 155, 61, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 138, 231, 206, 116, 217, 250, 37, 42, 137, 131, 151, 110, 171, 96, 13, 199, 89, 12, 119, 141, 4, 129, 52, 48, 132, 27, 198, 231, 101, 195, 76, 216, 26, 22, 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 138, 231, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 137, 28, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 96, 65, 43, 148, 229, 37, 202, 108, 237, 201, 245, 68, 253, 134, 247, 118, 6, 213, 47, 231, 49, 165, 208, 105, 219, 232, 54, 168, 191, 192, 251, 140, 145, 25, 99, 176, 174, 122, 20, 151, 31, 59, 70, 33, 191, 251, 128, 46, 240, 96, 83, 146, 185, 166, 200, 156, 127, 171, 29, 248, 99, 58, 90, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 69, 0, 194, 245, 33, 248, 63, 186, 94, 252, 43, 243, 239, 250, 174, 221, 228, 61, 10, 74, 223, 247, 133, 193, 33, 59, 113, 42, 58, 237, 13, 129, 87, 100, 42, 132, 50, 77, 176, 207, 150, 149, 235, 210, 119, 8, 212, 96, 142, 176, 51, 126, 13, 216, 123, 14, 67, 240, 250, 112, 199, 0, 217, 17}}, - expectedRetryable: false, - expectedError: nil, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := setupEVMRegistry(t) - hc := mocks.NewHttpClient(t) - - mr := MercuryV03Response{} - for i, blob := range tt.mockChainlinkBlobs { - r := MercuryV03Report{ - FeedID: tt.lookup.Feeds[i], - ValidFromTimestamp: 0, - ObservationsTimestamp: 0, - FullReport: blob, - } - mr.Reports = append(mr.Reports, r) - } - - b, err := json.Marshal(mr) - assert.Nil(t, err) - resp := &http.Response{ - StatusCode: tt.mockHttpStatusCode, - Body: io.NopCloser(bytes.NewReader(b)), - } - if tt.expectedError != nil && tt.expectedRetryable { - hc.On("Do", mock.Anything).Return(resp, nil).Times(totalAttempt) - } else { - hc.On("Do", mock.Anything).Return(resp, nil).Once() - } - r.hc = hc - - state, reason, values, retryable, ri, reqErr := r.doMercuryRequest(context.Background(), tt.lookup, tt.pluginRetryKey, r.lggr) - assert.Equal(t, tt.expectedValues, values) - assert.Equal(t, tt.expectedRetryable, retryable) - assert.Equal(t, tt.expectedRetryInterval, ri) - assert.Equal(t, tt.state, state) - assert.Equal(t, tt.reason, reason) - if tt.expectedError != nil { - assert.Equal(t, tt.expectedError.Error(), reqErr.Error()) - } - }) - } -} - -func TestEvmRegistry_SingleFeedRequest(t *testing.T) { - upkeepId := big.NewInt(123456789) - tests := []struct { - name string - index int - lookup *StreamsLookup - pluginRetryKey string - blob string - statusCode int - lastStatusCode int - retryNumber int - retryable bool - errorMessage string - }{ - { - name: "success - mercury responds in the first try", - index: 0, - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIdHex, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: blockNumber, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - blob: "0xab2123dc00000012", - }, - { - name: "success - retry for 404", - index: 0, - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIdHex, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: blockNumber, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - blob: "0xab2123dcbabbad", - retryNumber: 1, - statusCode: http.StatusNotFound, - lastStatusCode: http.StatusOK, - }, - { - name: "success - retry for 500", - index: 0, - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIdHex, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: blockNumber, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - blob: "0xab2123dcbbabad", - retryNumber: 2, - statusCode: http.StatusInternalServerError, - lastStatusCode: http.StatusOK, - }, - { - name: "failure - returns retryable", - index: 0, - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIdHex, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: blockNumber, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - blob: "0xab2123dc", - retryNumber: totalAttempt, - statusCode: http.StatusNotFound, - retryable: true, - errorMessage: "failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000: All attempts fail:\n#1: 404\n#2: 404\n#3: 404", - }, - { - name: "failure - returns retryable and then non-retryable", - index: 0, - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIdHex, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: blockNumber, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - blob: "0xab2123dc", - retryNumber: 1, - statusCode: http.StatusNotFound, - lastStatusCode: http.StatusTooManyRequests, - errorMessage: "failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000: All attempts fail:\n#1: 404\n#2: at block 123456 upkeep 123456789 received status code 429 for feed 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", - }, - { - name: "failure - returns not retryable", - index: 0, - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIdHex, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: blockNumber, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - blob: "0xab2123dc", - statusCode: http.StatusConflict, - errorMessage: "failed to request feed for 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000: All attempts fail:\n#1: at block 123456 upkeep 123456789 received status code 409 for feed 0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := setupEVMRegistry(t) - hc := mocks.NewHttpClient(t) - - mr := MercuryV02Response{ChainlinkBlob: tt.blob} - b, err := json.Marshal(mr) - assert.Nil(t, err) - - if tt.retryNumber == 0 { - if tt.errorMessage != "" { - resp := &http.Response{ - StatusCode: tt.statusCode, - Body: io.NopCloser(bytes.NewReader(b)), - } - hc.On("Do", mock.Anything).Return(resp, nil).Once() - } else { - resp := &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewReader(b)), - } - hc.On("Do", mock.Anything).Return(resp, nil).Once() - } - } else if tt.retryNumber > 0 && tt.retryNumber < totalAttempt { - retryResp := &http.Response{ - StatusCode: tt.statusCode, - Body: io.NopCloser(bytes.NewReader(b)), - } - hc.On("Do", mock.Anything).Return(retryResp, nil).Times(tt.retryNumber) - - resp := &http.Response{ - StatusCode: tt.lastStatusCode, - Body: io.NopCloser(bytes.NewReader(b)), - } - hc.On("Do", mock.Anything).Return(resp, nil).Once() - } else { - resp := &http.Response{ - StatusCode: tt.statusCode, - Body: io.NopCloser(bytes.NewReader(b)), - } - hc.On("Do", mock.Anything).Return(resp, nil).Times(tt.retryNumber) - } - r.hc = hc - - ch := make(chan MercuryData, 1) - r.singleFeedRequest(context.Background(), ch, tt.index, tt.lookup, r.lggr) - - m := <-ch - assert.Equal(t, tt.index, m.Index) - assert.Equal(t, tt.retryable, m.Retryable) - if tt.retryNumber >= totalAttempt || tt.errorMessage != "" { - assert.Equal(t, tt.errorMessage, m.Error.Error()) - assert.Equal(t, [][]byte{}, m.Bytes) - } else { - blobBytes, err := hexutil.Decode(tt.blob) - assert.Nil(t, err) - assert.Nil(t, m.Error) - assert.Equal(t, [][]byte{blobBytes}, m.Bytes) - } - }) - } -} - -func TestEvmRegistry_MultiFeedRequest(t *testing.T) { - upkeepId := big.NewInt(123456789) - tests := []struct { - name string - lookup *StreamsLookup - statusCode int - lastStatusCode int - pluginRetries int - pluginRetryKey string - retryNumber int - retryable bool - errorMessage string - firstResponse *MercuryV03Response - response *MercuryV03Response - }{ - { - name: "success - mercury responds in the first try", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIDs, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: timestamp, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - response: &MercuryV03Response{ - Reports: []MercuryV03Report{ - { - FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", - ValidFromTimestamp: 123456, - ObservationsTimestamp: 123456, - FullReport: "0xab2123dc00000012", - }, - { - FeedID: "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000", - ValidFromTimestamp: 123458, - ObservationsTimestamp: 123458, - FullReport: "0xab2123dc00000016", - }, - }, - }, - statusCode: http.StatusOK, - }, - { - name: "success - mercury responds in the first try with blocknumber", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIDs, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: blockNumber, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - response: &MercuryV03Response{ - Reports: []MercuryV03Report{ - { - FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", - ValidFromTimestamp: 123456, - ObservationsTimestamp: 123456, - FullReport: "0xab2123dc00000012", - }, - { - FeedID: "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000", - ValidFromTimestamp: 123458, - ObservationsTimestamp: 123458, - FullReport: "0xab2123dc00000016", - }, - }, - }, - statusCode: http.StatusOK, - }, - { - name: "success - retry 206", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIDs, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: timestamp, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - firstResponse: &MercuryV03Response{ - Reports: []MercuryV03Report{ - { - FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", - ValidFromTimestamp: 123456, - ObservationsTimestamp: 123456, - FullReport: "0xab2123dc00000012", - }, - }, - }, - response: &MercuryV03Response{ - Reports: []MercuryV03Report{ - { - FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", - ValidFromTimestamp: 123456, - ObservationsTimestamp: 123456, - FullReport: "0xab2123dc00000012", - }, - { - FeedID: "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000", - ValidFromTimestamp: 123458, - ObservationsTimestamp: 123458, - FullReport: "0xab2123dc00000019", - }, - }, - }, - retryNumber: 1, - statusCode: http.StatusPartialContent, - lastStatusCode: http.StatusOK, - }, - { - name: "success - retry for 500", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIDs, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: timestamp, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - retryNumber: 2, - statusCode: http.StatusInternalServerError, - lastStatusCode: http.StatusOK, - response: &MercuryV03Response{ - Reports: []MercuryV03Report{ - { - FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", - ValidFromTimestamp: 123456, - ObservationsTimestamp: 123456, - FullReport: "0xab2123dc00000012", - }, - { - FeedID: "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000", - ValidFromTimestamp: 123458, - ObservationsTimestamp: 123458, - FullReport: "0xab2123dc00000019", - }, - }, - }, - }, - { - name: "failure - fail to decode reportBlob", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIDs, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: timestamp, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - response: &MercuryV03Response{ - Reports: []MercuryV03Report{ - { - FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", - ValidFromTimestamp: 123456, - ObservationsTimestamp: 123456, - FullReport: "qerwiu", // invalid hex blob - }, - { - FeedID: "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000", - ValidFromTimestamp: 123458, - ObservationsTimestamp: 123458, - FullReport: "0xab2123dc00000016", - }, - }, - }, - statusCode: http.StatusOK, - retryable: false, - errorMessage: "All attempts fail:\n#1: hex string without 0x prefix", - }, - { - name: "failure - returns retryable with 1s plugin retry interval", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIDs, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: timestamp, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - retryNumber: totalAttempt, - statusCode: http.StatusInternalServerError, - retryable: true, - errorMessage: "All attempts fail:\n#1: 500\n#2: 500\n#3: 500", - }, - { - name: "failure - returns retryable with 5s plugin retry interval", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIDs, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: timestamp, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - pluginRetries: 6, - retryNumber: totalAttempt, - statusCode: http.StatusInternalServerError, - retryable: true, - errorMessage: "All attempts fail:\n#1: 500\n#2: 500\n#3: 500", - }, - { - name: "failure - returns retryable and then non-retryable", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIDs, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: timestamp, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - retryNumber: 1, - statusCode: http.StatusInternalServerError, - lastStatusCode: http.StatusUnauthorized, - errorMessage: "All attempts fail:\n#1: 500\n#2: at timestamp 123456 upkeep 123456789 received status code 401 from mercury v0.3, most likely this is caused by unauthorized upkeep", - }, - { - name: "failure - returns status code 422 not retryable", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIDs, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: timestamp, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - statusCode: http.StatusUnprocessableEntity, - errorMessage: "All attempts fail:\n#1: at timestamp 123456 upkeep 123456789 received status code 422 from mercury v0.3", - }, - { - name: "success - retry when reports length does not match feeds length", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIDs, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: timestamp, - Time: big.NewInt(123456), - }, - upkeepId: upkeepId, - }, - firstResponse: &MercuryV03Response{ - Reports: []MercuryV03Report{ - { - FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", - ValidFromTimestamp: 123456, - ObservationsTimestamp: 123456, - FullReport: "0xab2123dc00000012", - }, - }, - }, - response: &MercuryV03Response{ - Reports: []MercuryV03Report{ - { - FeedID: "0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", - ValidFromTimestamp: 123456, - ObservationsTimestamp: 123456, - FullReport: "0xab2123dc00000012", - }, - { - FeedID: "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000", - ValidFromTimestamp: 123458, - ObservationsTimestamp: 123458, - FullReport: "0xab2123dc00000019", - }, - }, - }, - retryNumber: 1, - statusCode: http.StatusOK, - lastStatusCode: http.StatusOK, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := setupEVMRegistry(t) - if tt.pluginRetries != 0 { - r.mercury.pluginRetryCache.Set(tt.pluginRetryKey, tt.pluginRetries, cache.DefaultExpiration) - } - - hc := mocks.NewHttpClient(t) - b, err := json.Marshal(tt.response) - assert.Nil(t, err) - - if tt.retryNumber == 0 { - resp := &http.Response{ - StatusCode: tt.statusCode, - Body: io.NopCloser(bytes.NewReader(b)), - } - hc.On("Do", mock.Anything).Return(resp, nil).Once() - } else if tt.retryNumber < totalAttempt { - if tt.firstResponse != nil && tt.response != nil { - b0, err := json.Marshal(tt.firstResponse) - assert.Nil(t, err) - resp0 := &http.Response{ - StatusCode: tt.statusCode, - Body: io.NopCloser(bytes.NewReader(b0)), - } - b1, err := json.Marshal(tt.response) - assert.Nil(t, err) - resp1 := &http.Response{ - StatusCode: tt.lastStatusCode, - Body: io.NopCloser(bytes.NewReader(b1)), - } - hc.On("Do", mock.Anything).Return(resp0, nil).Once().On("Do", mock.Anything).Return(resp1, nil).Once() - } else { - retryResp := &http.Response{ - StatusCode: tt.statusCode, - Body: io.NopCloser(bytes.NewReader(b)), - } - hc.On("Do", mock.Anything).Return(retryResp, nil).Times(tt.retryNumber) - - resp := &http.Response{ - StatusCode: tt.lastStatusCode, - Body: io.NopCloser(bytes.NewReader(b)), - } - hc.On("Do", mock.Anything).Return(resp, nil).Once() - } - } else { - resp := &http.Response{ - StatusCode: tt.statusCode, - Body: io.NopCloser(bytes.NewReader(b)), - } - hc.On("Do", mock.Anything).Return(resp, nil).Times(tt.retryNumber) - } - r.hc = hc - - ch := make(chan MercuryData, 1) - r.multiFeedsRequest(context.Background(), ch, tt.lookup, r.lggr) - - m := <-ch - assert.Equal(t, 0, m.Index) - assert.Equal(t, tt.retryable, m.Retryable) - if tt.retryNumber >= totalAttempt || tt.errorMessage != "" { - assert.Equal(t, tt.errorMessage, m.Error.Error()) - assert.Equal(t, [][]byte{}, m.Bytes) - } else { - assert.Nil(t, m.Error) - var reports [][]byte - for _, rsp := range tt.response.Reports { - b, _ := hexutil.Decode(rsp.FullReport) - reports = append(reports, b) - } - assert.Equal(t, reports, m.Bytes) - } - }) - } -} - -func TestEvmRegistry_CheckCallback(t *testing.T) { - upkeepId := big.NewInt(123456789) - bn := uint64(999) - bs := []byte{183, 114, 215, 10, 0, 0, 0, 0, 0, 0} - values := [][]byte{bs} - tests := []struct { - name string - lookup *StreamsLookup - values [][]byte - statusCode int - - callbackResp []byte - callbackErr error - - upkeepNeeded bool - performData []byte - wantErr assert.ErrorAssertionFunc - - state encoding.PipelineExecutionState - retryable bool - }{ - { - name: "success - empty extra data", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIdHex, - Feeds: []string{"ETD-USD", "BTC-ETH"}, - TimeParamKey: blockNumber, - Time: big.NewInt(100), - ExtraData: []byte{48, 120, 48, 48}, - }, - upkeepId: upkeepId, - block: bn, - }, - values: values, - statusCode: http.StatusOK, - callbackResp: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 48, 120, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - upkeepNeeded: true, - performData: []byte{48, 120, 48, 48}, - wantErr: assert.NoError, - }, - { - name: "success - with extra data", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIdHex, - Feeds: []string{"0x4554482d5553442d415242495452554d2d544553544e45540000000000000000", "0x4254432d5553442d415242495452554d2d544553544e45540000000000000000"}, - TimeParamKey: blockNumber, - Time: big.NewInt(18952430), - // this is the address of precompile contract ArbSys(0x0000000000000000000000000000000000000064) - ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, - }, - upkeepId: upkeepId, - block: bn, - }, - values: values, - statusCode: http.StatusOK, - callbackResp: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - upkeepNeeded: true, - performData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100}, - wantErr: assert.NoError, - }, - { - name: "failure - bad response", - lookup: &StreamsLookup{ - StreamsLookupError: &encoding.StreamsLookupError{ - FeedParamKey: feedIdHex, - Feeds: []string{"ETD-USD", "BTC-ETH"}, - TimeParamKey: blockNumber, - Time: big.NewInt(100), - ExtraData: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 48, 120, 48, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - }, - upkeepId: upkeepId, - block: bn, - }, - values: values, - statusCode: http.StatusOK, - callbackResp: []byte{}, - callbackErr: errors.New("bad response"), - wantErr: assert.Error, - state: encoding.RpcFlakyFailure, - retryable: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - client := new(evmClientMocks.Client) - r := setupEVMRegistry(t) - payload, err := r.abi.Pack("checkCallback", tt.lookup.upkeepId, values, tt.lookup.ExtraData) - require.Nil(t, err) - args := map[string]interface{}{ - "to": r.addr.Hex(), - "data": hexutil.Bytes(payload), - } - client.On("CallContext", mock.Anything, mock.AnythingOfType("*hexutil.Bytes"), "eth_call", args, hexutil.EncodeUint64(tt.lookup.block)).Return(tt.callbackErr). - Run(func(args mock.Arguments) { - by := args.Get(1).(*hexutil.Bytes) - *by = tt.callbackResp - }).Once() - r.client = client - - state, retryable, _, err := r.checkCallback(context.Background(), tt.values, tt.lookup) - tt.wantErr(t, err, fmt.Sprintf("Error asserion failed: %v", tt.name)) - assert.Equal(t, tt.state, state) - assert.Equal(t, tt.retryable, retryable) - }) - } -} diff --git a/core/services/ocr2/plugins/ocr2keeper/integration_21_test.go b/core/services/ocr2/plugins/ocr2keeper/integration_21_test.go index 562f972bc42..fcb3f6ebbb5 100644 --- a/core/services/ocr2/plugins/ocr2keeper/integration_21_test.go +++ b/core/services/ocr2/plugins/ocr2keeper/integration_21_test.go @@ -26,6 +26,8 @@ import ( "github.com/stretchr/testify/require" "github.com/umbracle/ethgo/abi" + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/streams" + "github.com/smartcontractkit/libocr/commontypes" "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper" "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper" @@ -33,6 +35,7 @@ import ( "github.com/smartcontractkit/ocr2keepers/pkg/v3/config" relaytypes "github.com/smartcontractkit/chainlink-relay/pkg/types" + "github.com/smartcontractkit/chainlink/v2/core/assets" "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller" automationForwarderLogic "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_forwarder_logic" @@ -52,7 +55,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/job" "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/ethkey" "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper" - evm21 "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21" "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm" ) @@ -884,7 +886,7 @@ func (c *feedLookupUpkeepController) EnableMercury( registry *iregistry21.IKeeperRegistryMaster, registryOwner *bind.TransactOpts, ) error { - adminBytes, _ := json.Marshal(evm21.UpkeepPrivilegeConfig{ + adminBytes, _ := json.Marshal(streams.UpkeepPrivilegeConfig{ MercuryEnabled: true, }) @@ -908,7 +910,7 @@ func (c *feedLookupUpkeepController) EnableMercury( return err } - var checkBytes evm21.UpkeepPrivilegeConfig + var checkBytes streams.UpkeepPrivilegeConfig if err := json.Unmarshal(bts, &checkBytes); err != nil { require.NoError(t, err) diff --git a/integration-tests/smoke/automation_test.go b/integration-tests/smoke/automation_test.go index 9e35b24df1e..d13cc0207fc 100644 --- a/integration-tests/smoke/automation_test.go +++ b/integration-tests/smoke/automation_test.go @@ -11,12 +11,16 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/common" "github.com/kelseyhightower/envconfig" + + "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21/mercury/streams" + + "github.com/ethereum/go-ethereum/common" "github.com/onsi/gomega" "github.com/stretchr/testify/require" "github.com/smartcontractkit/chainlink-testing-framework/blockchain" + "github.com/smartcontractkit/chainlink-testing-framework/logging" "github.com/smartcontractkit/chainlink-testing-framework/networks" @@ -24,8 +28,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/automation_utils_2_1" "github.com/smartcontractkit/chainlink/v2/core/store/models" - evm21 "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evm21" - "github.com/smartcontractkit/chainlink/integration-tests/actions" "github.com/smartcontractkit/chainlink/integration-tests/client" "github.com/smartcontractkit/chainlink/integration-tests/contracts" @@ -155,7 +157,7 @@ func SetupAutomationBasic(t *testing.T, nodeUpgrade bool) { if isMercury { // Set privilege config to enable mercury - privilegeConfigBytes, _ := json.Marshal(evm21.UpkeepPrivilegeConfig{ + privilegeConfigBytes, _ := json.Marshal(streams.UpkeepPrivilegeConfig{ MercuryEnabled: true, }) if err := registry.SetUpkeepPrivilegeConfig(upkeepIDs[i], privilegeConfigBytes); err != nil {