From 0c74d6f666798766684752df9b68f1b8b6564975 Mon Sep 17 00:00:00 2001 From: Austin <107539019+0xAustinWang@users.noreply.github.com> Date: Fri, 13 Dec 2024 10:57:09 +0800 Subject: [PATCH] Initialization for Crib (#15501) * wip * test crib integration flow * build failures * changes from ani's comments * some changes * revert isempty check * go lint and compare integers * fix types in test * check for error if there's already a feeds manager * check if job distributors exist first * wip changes * add retries to consistently failing JD cals * update retries with static duration * remove print statements * fix compile error * remove unnecessary mcms changes * build issues * passing deployment * clean up code comments * gomodtidy * lint * formatting strings * use the common changeset utilities when deploying home chain as well * compare nodes job distributors public key against the one we expect * move state reader under deployment module * add RPC type with internal and external rpcs --------- Co-authored-by: Radek Scheibinger --- deployment/address_book.go | 4 +- deployment/environment/crib/ccip_deployer.go | 136 ++++++++++++++++++ deployment/environment/crib/data.go | 81 +++++++++++ deployment/environment/crib/env.go | 45 ++++++ deployment/environment/crib/env_test.go | 18 +++ .../ccip-v2-scripts-address-book.json | 1 + .../ccip-v2-scripts-chains-details.json | 24 ++++ .../ccip-v2-scripts-nodes-details.json | 1 + deployment/environment/crib/types.go | 39 +++++ deployment/environment/devenv/don.go | 88 ++++++++---- .../environment/web/sdk/client/client.go | 16 ++- integration-tests/load/go.mod | 1 + integration-tests/testconfig/ccip/config.go | 22 ++- 13 files changed, 435 insertions(+), 41 deletions(-) create mode 100644 deployment/environment/crib/ccip_deployer.go create mode 100644 deployment/environment/crib/data.go create mode 100644 deployment/environment/crib/env.go create mode 100644 deployment/environment/crib/env_test.go create mode 100644 deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-address-book.json create mode 100644 deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-chains-details.json create mode 100644 deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-nodes-details.json create mode 100644 deployment/environment/crib/types.go diff --git a/deployment/address_book.go b/deployment/address_book.go index 6f605013011..3ce0332a4c3 100644 --- a/deployment/address_book.go +++ b/deployment/address_book.go @@ -89,8 +89,10 @@ type AddressBook interface { Remove(ab AddressBook) error } +type AddressesByChain map[uint64]map[string]TypeAndVersion + type AddressBookMap struct { - addressesByChain map[uint64]map[string]TypeAndVersion + addressesByChain AddressesByChain mtx sync.RWMutex } diff --git a/deployment/environment/crib/ccip_deployer.go b/deployment/environment/crib/ccip_deployer.go new file mode 100644 index 00000000000..aea7ad0cb8f --- /dev/null +++ b/deployment/environment/crib/ccip_deployer.go @@ -0,0 +1,136 @@ +package crib + +import ( + "context" + "errors" + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/smartcontractkit/ccip-owner-contracts/pkg/config" + commonchangeset "github.com/smartcontractkit/chainlink/deployment/common/changeset" + commontypes "github.com/smartcontractkit/chainlink/deployment/common/types" + "github.com/smartcontractkit/chainlink/deployment/environment/devenv" + "github.com/smartcontractkit/chainlink/v2/core/services/relay" + "math/big" + + "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/deployment/ccip/changeset" + "github.com/smartcontractkit/chainlink/v2/core/logger" +) + +// DeployHomeChainContracts deploys the home chain contracts so that the chainlink nodes can be started with the CR address in Capabilities.ExternalRegistry +// DeployHomeChainContracts is to 1. Set up crib with chains and chainlink nodes ( cap reg is not known yet so not setting the config with capreg address) +// Call DeployHomeChain changeset with nodeinfo ( the peer id and all) +func DeployHomeChainContracts(ctx context.Context, lggr logger.Logger, envConfig devenv.EnvironmentConfig, homeChainSel uint64, feedChainSel uint64) (deployment.CapabilityRegistryConfig, deployment.AddressBook, error) { + e, _, err := devenv.NewEnvironment(func() context.Context { return ctx }, lggr, envConfig) + if err != nil { + return deployment.CapabilityRegistryConfig{}, nil, err + } + if e == nil { + return deployment.CapabilityRegistryConfig{}, nil, errors.New("environment is nil") + } + + nodes, err := deployment.NodeInfo(e.NodeIDs, e.Offchain) + if err != nil { + return deployment.CapabilityRegistryConfig{}, e.ExistingAddresses, fmt.Errorf("failed to get node info from env: %w", err) + } + p2pIds := nodes.NonBootstraps().PeerIDs() + *e, err = commonchangeset.ApplyChangesets(nil, *e, nil, []commonchangeset.ChangesetApplication{ + { + Changeset: commonchangeset.WrapChangeSet(changeset.DeployHomeChain), + Config: changeset.DeployHomeChainConfig{ + HomeChainSel: homeChainSel, + RMNStaticConfig: changeset.NewTestRMNStaticConfig(), + RMNDynamicConfig: changeset.NewTestRMNDynamicConfig(), + NodeOperators: changeset.NewTestNodeOperator(e.Chains[homeChainSel].DeployerKey.From), + NodeP2PIDsPerNodeOpAdmin: map[string][][32]byte{ + "NodeOperator": p2pIds, + }, + }, + }, + }) + + state, err := changeset.LoadOnchainState(*e) + if err != nil { + return deployment.CapabilityRegistryConfig{}, e.ExistingAddresses, fmt.Errorf("failed to load on chain state: %w", err) + } + capRegAddr := state.Chains[homeChainSel].CapabilityRegistry.Address() + if capRegAddr == common.HexToAddress("0x") { + return deployment.CapabilityRegistryConfig{}, e.ExistingAddresses, fmt.Errorf("cap Reg address not found: %w", err) + } + capRegConfig := deployment.CapabilityRegistryConfig{ + EVMChainID: homeChainSel, + Contract: state.Chains[homeChainSel].CapabilityRegistry.Address(), + NetworkType: relay.NetworkEVM, + } + return capRegConfig, e.ExistingAddresses, nil +} + +func DeployCCIPAndAddLanes(ctx context.Context, lggr logger.Logger, envConfig devenv.EnvironmentConfig, homeChainSel, feedChainSel uint64, ab deployment.AddressBook) (DeployCCIPOutput, error) { + e, _, err := devenv.NewEnvironment(func() context.Context { return ctx }, lggr, envConfig) + if err != nil { + return DeployCCIPOutput{}, fmt.Errorf("failed to initiate new environment: %w", err) + } + e.ExistingAddresses = ab + allChainIds := e.AllChainSelectors() + cfg := make(map[uint64]commontypes.MCMSWithTimelockConfig) + for _, chain := range e.AllChainSelectors() { + mcmsConfig, err := config.NewConfig(1, []common.Address{e.Chains[chain].DeployerKey.From}, []config.Config{}) + if err != nil { + return DeployCCIPOutput{}, fmt.Errorf("failed to create mcms config: %w", err) + } + cfg[chain] = commontypes.MCMSWithTimelockConfig{ + Canceller: *mcmsConfig, + Bypasser: *mcmsConfig, + Proposer: *mcmsConfig, + TimelockMinDelay: big.NewInt(0), + } + } + + // This will not apply any proposals because we pass nil to testing. + // However, setup is ok because we only need to deploy the contracts and distribute job specs + *e, err = commonchangeset.ApplyChangesets(nil, *e, nil, []commonchangeset.ChangesetApplication{ + { + Changeset: commonchangeset.WrapChangeSet(commonchangeset.DeployLinkToken), + Config: allChainIds, + }, + { + Changeset: commonchangeset.WrapChangeSet(changeset.DeployPrerequisites), + Config: changeset.DeployPrerequisiteConfig{ + ChainSelectors: allChainIds, + }, + }, + { + Changeset: commonchangeset.WrapChangeSet(commonchangeset.DeployMCMSWithTimelock), + Config: cfg, + }, + { + Changeset: commonchangeset.WrapChangeSet(changeset.DeployChainContracts), + Config: changeset.DeployChainContractsConfig{ + ChainSelectors: allChainIds, + HomeChainSelector: homeChainSel, + }, + }, + { + Changeset: commonchangeset.WrapChangeSet(changeset.CCIPCapabilityJobspec), + Config: struct{}{}, + }, + }) + state, err := changeset.LoadOnchainState(*e) + if err != nil { + return DeployCCIPOutput{}, fmt.Errorf("failed to load onchain state: %w", err) + } + // Add all lanes + err = changeset.AddLanesForAll(*e, state) + if err != nil { + return DeployCCIPOutput{}, fmt.Errorf("failed to add lanes: %w", err) + } + + addresses, err := e.ExistingAddresses.Addresses() + if err != nil { + return DeployCCIPOutput{}, fmt.Errorf("failed to get convert address book to address book map: %w", err) + } + return DeployCCIPOutput{ + AddressBook: *deployment.NewMemoryAddressBookFromMap(addresses), + NodeIDs: e.NodeIDs, + }, err +} diff --git a/deployment/environment/crib/data.go b/deployment/environment/crib/data.go new file mode 100644 index 00000000000..b9197691613 --- /dev/null +++ b/deployment/environment/crib/data.go @@ -0,0 +1,81 @@ +package crib + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/deployment/environment/devenv" +) + +type OutputReader struct { + outputDir string +} + +func NewOutputReader(outputDir string) *OutputReader { + return &OutputReader{outputDir: outputDir} +} + +func (r *OutputReader) ReadNodesDetails() NodesDetails { + byteValue := r.readFile(NodesDetailsFileName) + + var result NodesDetails + + // Unmarshal the JSON into the map + err := json.Unmarshal(byteValue, &result) + if err != nil { + fmt.Println("Error unmarshalling JSON:", err) + panic(err) + } + + return result +} + +func (r *OutputReader) ReadChainConfigs() []devenv.ChainConfig { + byteValue := r.readFile(ChainsConfigsFileName) + + var result []devenv.ChainConfig + + // Unmarshal the JSON into the map + err := json.Unmarshal(byteValue, &result) + if err != nil { + fmt.Println("Error unmarshalling JSON:", err) + panic(err) + } + + return result +} + +func (r *OutputReader) ReadAddressBook() *deployment.AddressBookMap { + byteValue := r.readFile(AddressBookFileName) + + var result map[uint64]map[string]deployment.TypeAndVersion + + // Unmarshal the JSON into the map + err := json.Unmarshal(byteValue, &result) + if err != nil { + fmt.Println("Error unmarshalling JSON:", err) + panic(err) + } + + return deployment.NewMemoryAddressBookFromMap(result) +} + +func (r *OutputReader) readFile(fileName string) []byte { + file, err := os.Open(fmt.Sprintf("%s/%s", r.outputDir, fileName)) + if err != nil { + fmt.Println("Error opening file:", err) + panic(err) + } + defer file.Close() + + // Read the file's content into a byte slice + byteValue, err := io.ReadAll(file) + if err != nil { + fmt.Println("Error reading file:", err) + panic(err) + } + return byteValue +} diff --git a/deployment/environment/crib/env.go b/deployment/environment/crib/env.go new file mode 100644 index 00000000000..3af1acaf754 --- /dev/null +++ b/deployment/environment/crib/env.go @@ -0,0 +1,45 @@ +package crib + +const ( + AddressBookFileName = "ccip-v2-scripts-address-book.json" + NodesDetailsFileName = "ccip-v2-scripts-nodes-details.json" + ChainsConfigsFileName = "ccip-v2-scripts-chains-details.json" +) + +type CRIBEnv struct { + envStateDir string +} + +func NewDevspaceEnvFromStateDir(envStateDir string) CRIBEnv { + return CRIBEnv{ + envStateDir: envStateDir, + } +} + +func (c CRIBEnv) GetConfig() DeployOutput { + reader := NewOutputReader(c.envStateDir) + nodesDetails := reader.ReadNodesDetails() + chainConfigs := reader.ReadChainConfigs() + return DeployOutput{ + AddressBook: reader.ReadAddressBook(), + NodeIDs: nodesDetails.NodeIDs, + Chains: chainConfigs, + } +} + +type RPC struct { + External *string + Internal *string +} + +type ChainConfig struct { + ChainID uint64 // chain id as per EIP-155, mainly applicable for EVM chains + ChainName string // name of the chain populated from chainselector repo + ChainType string // should denote the chain family. Acceptable values are EVM, COSMOS, SOLANA, STARKNET, APTOS etc + WSRPCs []RPC // websocket rpcs to connect to the chain + HTTPRPCs []RPC // http rpcs to connect to the chain +} + +type NodesDetails struct { + NodeIDs []string +} diff --git a/deployment/environment/crib/env_test.go b/deployment/environment/crib/env_test.go new file mode 100644 index 00000000000..262a2540923 --- /dev/null +++ b/deployment/environment/crib/env_test.go @@ -0,0 +1,18 @@ +package crib + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestShouldProvideEnvironmentConfig(t *testing.T) { + t.Parallel() + env := NewDevspaceEnvFromStateDir("testdata/lanes-deployed-state") + config := env.GetConfig() + require.NotNil(t, config) + assert.NotEmpty(t, config.NodeIDs) + assert.NotNil(t, config.AddressBook) + assert.NotEmpty(t, config.Chains) +} diff --git a/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-address-book.json b/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-address-book.json new file mode 100644 index 00000000000..e4b2672cb5f --- /dev/null +++ b/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-address-book.json @@ -0,0 +1 @@ +{"12922642891491394802":{"0x05Aa229Aec102f78CE0E852A812a388F076Aa555":{"Type":"CancellerManyChainMultiSig","Version":"1.0.0"},"0x0D4ff719551E23185Aeb16FFbF2ABEbB90635942":{"Type":"TestRouter","Version":"1.2.0"},"0x0f5D1ef48f12b6f691401bfe88c2037c690a6afe":{"Type":"ProposerManyChainMultiSig","Version":"1.0.0"},"0x2dE080e97B0caE9825375D31f5D0eD5751fDf16D":{"Type":"CCIPReceiver","Version":"1.0.0"},"0x2fc631e4B3018258759C52AF169200213e84ABab":{"Type":"OnRamp","Version":"1.6.0-dev"},"0x5C7c905B505f0Cf40Ab6600d05e677F717916F6B":{"Type":"Router","Version":"1.2.0"},"0x63cf2Cd54fE91e3545D1379abf5bfd194545259d":{"Type":"OffRamp","Version":"1.6.0-dev"},"0x712516e61C8B383dF4A63CFe83d7701Bce54B03e":{"Type":"LinkToken","Version":"1.0.0"},"0x71C95911E9a5D330f4D621842EC243EE1343292e":{"Type":"PriceFeed","Version":"1.0.0"},"0x73eccD6288e117cAcA738BDAD4FEC51312166C1A":{"Type":"RMNRemote","Version":"1.6.0-dev"},"0x8464135c8F25Da09e49BC8782676a84730C318bC":{"Type":"PriceFeed","Version":"1.0.0"},"0x85C5Dd61585773423e378146D4bEC6f8D149E248":{"Type":"TokenAdminRegistry","Version":"1.5.0"},"0x948B3c65b89DF0B4894ABE91E6D02FE579834F8F":{"Type":"WETH9","Version":"1.0.0"},"0xAfe1b5bdEbD4ae65AF2024738bf0735fbb65d44b":{"Type":"FeeQuoter","Version":"1.6.0-dev"},"0xC6bA8C3233eCF65B761049ef63466945c362EdD2":{"Type":"BypasserManyChainMultiSig","Version":"1.0.0"},"0xbCF26943C0197d2eE0E5D05c716Be60cc2761508":{"Type":"AdminManyChainMultiSig","Version":"1.0.0"},"0xcA03Dc4665A8C3603cb4Fd5Ce71Af9649dC00d44":{"Type":"RBACTimelock","Version":"1.0.0"},"0xe6b98F104c1BEf218F3893ADab4160Dc73Eb8367":{"Type":"ARMProxy","Version":"1.0.0"},"0xfbAb4aa40C202E4e80390171E82379824f7372dd":{"Type":"NonceManager","Version":"1.6.0-dev"}},"3379446385462418246":{"0x09635F643e140090A9A8Dcd712eD6285858ceBef":{"Type":"RMNRemote","Version":"1.6.0-dev"},"0x0B306BF915C4d645ff596e518fAf3F9669b97016":{"Type":"LinkToken","Version":"1.0.0"},"0x1613beB3B2C4f22Ee086B2b38C1476A3cE7f78E8":{"Type":"OnRamp","Version":"1.6.0-dev"},"0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6":{"Type":"CCIPHome","Version":"1.6.0-dev"},"0x322813Fd9A801c5507c9de605d63CEA4f2CE6c44":{"Type":"ProposerManyChainMultiSig","Version":"1.0.0"},"0x3Aa5ebB10DC797CAC828524e59A333d0A371443c":{"Type":"BypasserManyChainMultiSig","Version":"1.0.0"},"0x4A679253410272dd5232B3Ff7cF5dbB88f295319":{"Type":"RBACTimelock","Version":"1.0.0"},"0x59b670e9fA9D0A427751Af201D676719a970857b":{"Type":"CancellerManyChainMultiSig","Version":"1.0.0"},"0x67d269191c92Caf3cD7723F116c85e6E9bf55933":{"Type":"ARMProxy","Version":"1.0.0"},"0x7a2088a1bFc9d81c55368AE168C2C02570cB814F":{"Type":"CCIPReceiver","Version":"1.0.0"},"0x84eA74d481Ee0A5332c457a4d796187F6Ba67fEB":{"Type":"TokenAdminRegistry","Version":"1.5.0"},"0x851356ae760d987E095750cCeb3bC6014560891C":{"Type":"OffRamp","Version":"1.6.0-dev"},"0x8A791620dd6260079BF849Dc5567aDC3F2FdC318":{"Type":"RMNHome","Version":"1.6.0-dev"},"0x9A676e781A523b5d0C0e43731313A708CB607508":{"Type":"WETH9","Version":"1.0.0"},"0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE":{"Type":"AdminManyChainMultiSig","Version":"1.0.0"},"0x9E545E3C0baAB3E08CdfD552C960A1050f373042":{"Type":"NonceManager","Version":"1.6.0-dev"},"0xE6E340D132b5f46d1e472DebcD681B2aBc16e57E":{"Type":"Router","Version":"1.2.0"},"0xa513E6E4b8f2a923D98304ec87F64353C4D5C853":{"Type":"CapabilitiesRegistry","Version":"1.0.0"},"0xa82fF9aFd8f496c3d6ac40E2a0F282E47488CFc9":{"Type":"FeeQuoter","Version":"1.6.0-dev"},"0xc3e53F4d16Ae77Db1c982e75a937B9f60FE63690":{"Type":"TestRouter","Version":"1.2.0"}}} diff --git a/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-chains-details.json b/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-chains-details.json new file mode 100644 index 00000000000..f93ea4ce231 --- /dev/null +++ b/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-chains-details.json @@ -0,0 +1,24 @@ +[ + { + "ChainID": 1337, + "ChainName": "alpha", + "ChainType": "EVM", + "WSRPCs": [ + "wss://crib-local-geth-1337-ws.local:443" + ], + "HTTPRPCs": [ + "https://crib-local-geth-1337-ws.local:443" + ] + }, + { + "ChainID": 2337, + "ChainName": "alpha", + "ChainType": "EVM", + "WSRPCs": [ + "wss://crib-local-geth-2337-ws.local:443" + ], + "HTTPRPCs": [ + "https://crib-local-geth-2337-ws.local:443" + ] + } +] diff --git a/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-nodes-details.json b/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-nodes-details.json new file mode 100644 index 00000000000..477ae0527b1 --- /dev/null +++ b/deployment/environment/crib/testdata/lanes-deployed-state/ccip-v2-scripts-nodes-details.json @@ -0,0 +1 @@ +{"NodeIDs":["node_2URuou3RXmtZu5gLQX8qd","node_m9TTQbUxBx3WjDEjmpVDL","node_4FiKVPtuQjCTvHnS7QpES","node_A4VTgecDwMoG2YYicyjuG","node_jQFpzXDadzaADq147nThS"]} diff --git a/deployment/environment/crib/types.go b/deployment/environment/crib/types.go new file mode 100644 index 00000000000..d19c8424443 --- /dev/null +++ b/deployment/environment/crib/types.go @@ -0,0 +1,39 @@ +package crib + +import ( + "context" + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink/deployment" + "github.com/smartcontractkit/chainlink/deployment/environment/devenv" +) + +const ( + CRIB_ENV_NAME = "Crib Environment" +) + +type DeployOutput struct { + NodeIDs []string + Chains []devenv.ChainConfig // chain selector -> Chain Config + AddressBook deployment.AddressBook // Addresses of all contracts +} + +type DeployCCIPOutput struct { + AddressBook deployment.AddressBookMap + NodeIDs []string +} + +func NewDeployEnvironmentFromCribOutput(lggr logger.Logger, output DeployOutput) (*deployment.Environment, error) { + chains, err := devenv.NewChains(lggr, output.Chains) + if err != nil { + return nil, err + } + return deployment.NewEnvironment( + CRIB_ENV_NAME, + lggr, + output.AddressBook, + chains, + output.NodeIDs, + nil, // todo: populate the offchain client using output.DON + func() context.Context { return context.Background() }, deployment.XXXGenerateTestOCRSecrets(), + ), nil +} diff --git a/deployment/environment/devenv/don.go b/deployment/environment/devenv/don.go index 05a3d5bea08..76f6ee92b68 100644 --- a/deployment/environment/devenv/don.go +++ b/deployment/environment/devenv/don.go @@ -2,7 +2,9 @@ package devenv import ( "context" + "errors" "fmt" + chainsel "github.com/smartcontractkit/chain-selectors" "strconv" "strings" "time" @@ -10,8 +12,6 @@ import ( "github.com/hashicorp/go-multierror" "github.com/rs/zerolog" "github.com/sethvargo/go-retry" - chainsel "github.com/smartcontractkit/chain-selectors" - nodev1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/node" clclient "github.com/smartcontractkit/chainlink/deployment/environment/nodeclient" "github.com/smartcontractkit/chainlink/deployment/environment/web/sdk/client" @@ -185,7 +185,7 @@ type JDChainConfigInput struct { // It expects bootstrap nodes to have label with key "type" and value as "bootstrap". // It fetches the account address, peer id, and OCR2 key bundle id and creates the JobDistributorChainConfig. func (n *Node) CreateCCIPOCRSupportedChains(ctx context.Context, chains []JDChainConfigInput, jd JobDistributor) error { - for i, chain := range chains { + for _, chain := range chains { chainId := strconv.FormatUint(chain.ChainID, 10) var account string switch chain.ChainType { @@ -239,35 +239,51 @@ func (n *Node) CreateCCIPOCRSupportedChains(ctx context.Context, chains []JDChai break } } - // JD silently fails to update nodeChainConfig. Therefore, we fetch the node config and - // if it's not updated , throw an error - _, err = n.gqlClient.CreateJobDistributorChainConfig(ctx, client.JobDistributorChainConfigInput{ - JobDistributorID: n.JDId, - ChainID: chainId, - ChainType: chain.ChainType, - AccountAddr: account, - AdminAddr: n.adminAddr, - Ocr2Enabled: true, - Ocr2IsBootstrap: isBootstrap, - Ocr2Multiaddr: n.multiAddr, - Ocr2P2PPeerID: value(peerID), - Ocr2KeyBundleID: ocr2BundleId, - Ocr2Plugins: `{"commit":true,"execute":true,"median":false,"mercury":false}`, + + // retry twice with 5 seconds interval to create JobDistributorChainConfig + err = retry.Do(ctx, retry.WithMaxDuration(10*time.Second, retry.NewConstant(3*time.Second)), func(ctx context.Context) error { + // check the node chain config to see if this chain already exists + nodeChainConfigs, err := jd.ListNodeChainConfigs(context.Background(), &nodev1.ListNodeChainConfigsRequest{ + Filter: &nodev1.ListNodeChainConfigsRequest_Filter{ + NodeIds: []string{n.NodeId}, + }}) + if err != nil { + return retry.RetryableError(fmt.Errorf("failed to list node chain configs for node %s, retrying..: %w", n.Name, err)) + } + if nodeChainConfigs != nil { + for _, chainConfig := range nodeChainConfigs.ChainConfigs { + if chainConfig.Chain.Id == chainId { + return nil + } + } + } + + // JD silently fails to update nodeChainConfig. Therefore, we fetch the node config and + // if it's not updated , throw an error + _, err = n.gqlClient.CreateJobDistributorChainConfig(ctx, client.JobDistributorChainConfigInput{ + JobDistributorID: n.JDId, + ChainID: chainId, + ChainType: chain.ChainType, + AccountAddr: account, + AdminAddr: n.adminAddr, + Ocr2Enabled: true, + Ocr2IsBootstrap: isBootstrap, + Ocr2Multiaddr: n.multiAddr, + Ocr2P2PPeerID: value(peerID), + Ocr2KeyBundleID: ocr2BundleId, + Ocr2Plugins: `{"commit":true,"execute":true,"median":false,"mercury":false}`, + }) + // todo: add a check if the chain config failed because of a duplicate in that case, should we update or return success? + if err != nil { + return fmt.Errorf("failed to create CCIPOCR2SupportedChains for node %s: %w", n.Name, err) + } + + return retry.RetryableError(errors.New("retrying CreateChainConfig in JD")) }) + if err != nil { return fmt.Errorf("failed to create CCIPOCR2SupportedChains for node %s: %w", n.Name, err) } - // query the node chain config to check if it's created - nodeChainConfigs, err := jd.ListNodeChainConfigs(context.Background(), &nodev1.ListNodeChainConfigsRequest{ - Filter: &nodev1.ListNodeChainConfigsRequest_Filter{ - NodeIds: []string{n.NodeId}, - }}) - if err != nil { - return fmt.Errorf("failed to list node chain configs for node %s: %w", n.Name, err) - } - if nodeChainConfigs == nil || len(nodeChainConfigs.ChainConfigs) < i+1 { - return fmt.Errorf("failed to create chain config for node %s", n.Name) - } } return nil } @@ -377,6 +393,17 @@ func (n *Node) CreateJobDistributor(ctx context.Context, jd JobDistributor) (str return "", err } // create the job distributor in the node with the csa key + resp, err := n.gqlClient.ListJobDistributors(ctx) + if err != nil { + return "", fmt.Errorf("could not list job distrubutors: %w", err) + } + if len(resp.FeedsManagers.Results) > 0 { + for _, fm := range resp.FeedsManagers.Results { + if fm.GetPublicKey() == csaKey { + return fm.GetId(), nil + } + } + } return n.gqlClient.CreateJobDistributor(ctx, client.JobDistributorInput{ Name: "Job Distributor", Uri: jd.WSRPC, @@ -394,8 +421,9 @@ func (n *Node) SetUpAndLinkJobDistributor(ctx context.Context, jd JobDistributor } // now create the job distributor in the node id, err := n.CreateJobDistributor(ctx, jd) - if err != nil && !strings.Contains(err.Error(), "DuplicateFeedsManagerError") { - return err + if err != nil && + (!strings.Contains(err.Error(), "only a single feeds manager is supported") || !strings.Contains(err.Error(), "DuplicateFeedsManagerError")) { + return fmt.Errorf("failed to create job distributor in node %s: %w", n.Name, err) } // wait for the node to connect to the job distributor err = retry.Do(ctx, retry.WithMaxDuration(1*time.Minute, retry.NewFibonacci(1*time.Second)), func(ctx context.Context) error { diff --git a/deployment/environment/web/sdk/client/client.go b/deployment/environment/web/sdk/client/client.go index 5472591ef94..e0a56b9e642 100644 --- a/deployment/environment/web/sdk/client/client.go +++ b/deployment/environment/web/sdk/client/client.go @@ -4,10 +4,11 @@ import ( "context" "encoding/json" "fmt" + "github.com/Khan/genqlient/graphql" + "github.com/sethvargo/go-retry" "net/http" "strings" - - "github.com/Khan/genqlient/graphql" + "time" "github.com/smartcontractkit/chainlink/deployment/environment/web/sdk/client/doer" "github.com/smartcontractkit/chainlink/deployment/environment/web/sdk/internal/generated" @@ -60,8 +61,15 @@ func New(baseURI string, creds Credentials) (Client, error) { endpoints: ep, credentials: creds, } - - if err := c.login(); err != nil { + + err := retry.Do(context.Background(), retry.WithMaxDuration(10*time.Second, retry.NewFibonacci(2*time.Second)), func(ctx context.Context) error { + err := c.login() + if err != nil { + return retry.RetryableError(fmt.Errorf("retrying login to node: %w", err)) + } + return nil + }) + if err != nil { return nil, fmt.Errorf("failed to login to node: %w", err) } diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 3d240cccc9e..c94ea489c1b 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -398,6 +398,7 @@ require ( github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/segmentio/ksuid v1.0.4 // indirect github.com/sercand/kuberesolver/v5 v5.1.1 // indirect + github.com/sethvargo/go-retry v0.2.4 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shirou/gopsutil/v3 v3.24.3 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect diff --git a/integration-tests/testconfig/ccip/config.go b/integration-tests/testconfig/ccip/config.go index 72c81f05f47..70c850fd591 100644 --- a/integration-tests/testconfig/ccip/config.go +++ b/integration-tests/testconfig/ccip/config.go @@ -147,6 +147,9 @@ func (o *JDConfig) GetJDDBVersion() string { func (o *Config) Validate() error { var chainIds []int64 for _, net := range o.PrivateEthereumNetworks { + if net.EthereumChainConfig.ChainID < 0 { + return fmt.Errorf("negative chain ID found for network %d", net.EthereumChainConfig.ChainID) + } chainIds = append(chainIds, int64(net.EthereumChainConfig.ChainID)) } homeChainSelector, err := strconv.ParseUint(pointer.GetString(o.HomeChainSelector), 10, 64) @@ -189,14 +192,21 @@ func IsSelectorValid(selector uint64, chainIds []int64) (bool, error) { if err != nil { return false, err } - if chainId >= math.MaxInt64 { - return false, fmt.Errorf("chain id overflows int64: %d", chainId) - } - expId := int64(chainId) - for _, id := range chainIds { - if id == expId { + + for _, cID := range chainIds { + if isEqualUint64AndInt64(chainId, cID) { return true, nil } } return false, nil } + +func isEqualUint64AndInt64(u uint64, i int64) bool { + if i < 0 { + return false // uint64 cannot be equal to a negative int64 + } + if u > math.MaxInt64 { + return false // uint64 cannot be equal to an int64 if it exceeds the maximum int64 value + } + return u == uint64(i) +}