From ac6c9496fc63af978e0ccc4c4185dbd4f45f3ccf Mon Sep 17 00:00:00 2001 From: krehermann <16602512+krehermann@users.noreply.github.com> Date: Fri, 25 Oct 2024 10:00:02 -0600 Subject: [PATCH] keystone deployment fixes (#14928) * hacking * wip * wip * working with more logging * more logging & fix signature bug * stage working * linter * address comments * fix test --- .../keystone/changeset/deploy_ocr3.go | 29 +++---- .../keystone/changeset/deploy_ocr3_test.go | 9 +- .../deployment/keystone/deploy.go | 84 ++++++++++++++++--- .../deployment/keystone/types.go | 80 ++++++++++++------ .../deployment/keystone/types_test.go | 22 +++-- 5 files changed, 153 insertions(+), 71 deletions(-) diff --git a/integration-tests/deployment/keystone/changeset/deploy_ocr3.go b/integration-tests/deployment/keystone/changeset/deploy_ocr3.go index 51566787bd0..3b7e81460fb 100644 --- a/integration-tests/deployment/keystone/changeset/deploy_ocr3.go +++ b/integration-tests/deployment/keystone/changeset/deploy_ocr3.go @@ -5,36 +5,29 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink/integration-tests/deployment" + "github.com/smartcontractkit/chainlink/integration-tests/deployment/clo/models" kslib "github.com/smartcontractkit/chainlink/integration-tests/deployment/keystone" ) func DeployOCR3(lggr logger.Logger, env deployment.Environment, ab deployment.AddressBook, registryChainSel uint64) (deployment.ChangesetOutput, error) { - // must have capabilities registry deployed - regAddrs, err := ab.AddressesForChain(registryChainSel) - if err != nil { - return deployment.ChangesetOutput{}, fmt.Errorf("no addresses found for chain %d: %w", registryChainSel, err) - } - found := false - for _, addr := range regAddrs { - if addr.Type == kslib.CapabilityRegistryTypeVersion.Type { - found = true - break - } - } - if !found { - return deployment.ChangesetOutput{}, fmt.Errorf("no capabilities registry found for changeset %s", "0001_deploy_registry") - } - // ocr3 only deployed on registry chain c, ok := env.Chains[registryChainSel] if !ok { return deployment.ChangesetOutput{}, fmt.Errorf("chain not found in environment") } - err = kslib.DeployOCR3(lggr, c, ab) + err := kslib.DeployOCR3(lggr, c, ab) if err != nil { return deployment.ChangesetOutput{}, fmt.Errorf("failed to deploy OCR3Capability: %w", err) } - return deployment.ChangesetOutput{AddressBook: ab}, nil +} + +func ConfigureOCR3Contract(lggr logger.Logger, env deployment.Environment, ab deployment.AddressBook, registryChainSel uint64, nodes []*models.Node, cfg kslib.OracleConfigSource) (deployment.ChangesetOutput, error) { + + err := kslib.ConfigureOCR3ContractFromCLO(&env, registryChainSel, nodes, ab, &cfg) + if err != nil { + return deployment.ChangesetOutput{}, fmt.Errorf("failed to configure OCR3Capability: %w", err) + } + return deployment.ChangesetOutput{AddressBook: ab}, nil } diff --git a/integration-tests/deployment/keystone/changeset/deploy_ocr3_test.go b/integration-tests/deployment/keystone/changeset/deploy_ocr3_test.go index eb7e34e7bcd..b4b8f62f1d6 100644 --- a/integration-tests/deployment/keystone/changeset/deploy_ocr3_test.go +++ b/integration-tests/deployment/keystone/changeset/deploy_ocr3_test.go @@ -10,7 +10,6 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink/integration-tests/deployment" - kslb "github.com/smartcontractkit/chainlink/integration-tests/deployment/keystone" "github.com/smartcontractkit/chainlink/integration-tests/deployment/keystone/changeset" "github.com/smartcontractkit/chainlink/integration-tests/deployment/memory" ) @@ -26,20 +25,14 @@ func TestDeployOCR3(t *testing.T) { env := memory.NewMemoryEnvironment(t, lggr, zapcore.DebugLevel, cfg) registrySel := env.AllChainSelectors()[0] - // err if no capabilities registry on chain 0 - _, err := changeset.DeployOCR3(lggr, env, ab, registrySel) - require.Error(t, err) - // fake capabilities registry - err = ab.Save(registrySel, "0x0000000000000000000000000000000000000001", kslb.CapabilityRegistryTypeVersion) - require.NoError(t, err) resp, err := changeset.DeployOCR3(lggr, env, ab, registrySel) require.NoError(t, err) require.NotNil(t, resp) // OCR3 should be deployed on chain 0 addrs, err := resp.AddressBook.AddressesForChain(registrySel) require.NoError(t, err) - require.Len(t, addrs, 2) + require.Len(t, addrs, 1) // nothing on chain 1 require.NotEqual(t, registrySel, env.AllChainSelectors()[1]) diff --git a/integration-tests/deployment/keystone/deploy.go b/integration-tests/deployment/keystone/deploy.go index 204247fd721..71babbb2180 100644 --- a/integration-tests/deployment/keystone/deploy.go +++ b/integration-tests/deployment/keystone/deploy.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/rpc" "github.com/smartcontractkit/chainlink/integration-tests/deployment" + "github.com/smartcontractkit/chainlink/integration-tests/deployment/clo/models" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" @@ -94,7 +95,7 @@ func ConfigureContracts(ctx context.Context, lggr logger.Logger, req ConfigureCo } // now we have the capability registry set up we need to configure the forwarder contracts and the OCR3 contract - dons, err := joinInfoAndNodes(cfgRegistryResp.DonInfos, req.Dons) + dons, err := joinInfoAndNodes(cfgRegistryResp.DonInfos, req.Dons, req.RegistryChainSel) if err != nil { return nil, fmt.Errorf("failed to assimilate registry to Dons: %w", err) } @@ -170,7 +171,7 @@ func ConfigureRegistry(ctx context.Context, lggr logger.Logger, req ConfigureCon // all the subsequent calls to the registry are in terms of nodes // compute the mapping of dons to their nodes for reuse in various registry calls - donToOcr2Nodes, err := mapDonsToNodes(req.Dons, true) + donToOcr2Nodes, err := mapDonsToNodes(req.Dons, true, req.RegistryChainSel) if err != nil { return nil, fmt.Errorf("failed to map dons to nodes: %w", err) } @@ -267,7 +268,7 @@ func ConfigureForwardContracts(env *deployment.Environment, dons []RegisteredDon return fmt.Errorf("no forwarder contract found for chain %d", chain.Selector) } - err := configureForwarder(chain, fwrd, dons) + err := configureForwarder(env.Logger, chain, fwrd, dons) if err != nil { return fmt.Errorf("failed to configure forwarder for chain selector %d: %w", chain.Selector, err) } @@ -301,14 +302,14 @@ func ConfigureOCR3Contract(env *deployment.Environment, chainSel uint64, dons [] } contract := contracts.OCR3 if contract == nil { - return fmt.Errorf("no forwarder contract found for chain %d", chainSel) + return fmt.Errorf("no ocr3 contract found for chain %d", chainSel) } _, err := configureOCR3contract(configureOCR3Request{ cfg: cfg, chain: registryChain, contract: contract, - don: don, + nodes: don.Nodes, }) if err != nil { return fmt.Errorf("failed to configure OCR3 contract for don %s: %w", don.Name, err) @@ -317,6 +318,43 @@ func ConfigureOCR3Contract(env *deployment.Environment, chainSel uint64, dons [] return nil } +func ConfigureOCR3ContractFromCLO(env *deployment.Environment, chainSel uint64, nodes []*models.Node, addrBook deployment.AddressBook, cfg *OracleConfigSource) error { + registryChain, ok := env.Chains[chainSel] + if !ok { + return fmt.Errorf("chain %d not found in environment", chainSel) + } + contractSetsResp, err := GetContractSets(&GetContractSetsRequest{ + Chains: env.Chains, + AddressBook: addrBook, + }) + if err != nil { + return fmt.Errorf("failed to get contract sets: %w", err) + } + contracts, ok := contractSetsResp.ContractSets[chainSel] + if !ok { + return fmt.Errorf("failed to get contract set for chain %d", chainSel) + } + contract := contracts.OCR3 + if contract == nil { + return fmt.Errorf("no ocr3 contract found for chain %d", chainSel) + } + var ocr2nodes []*ocr2Node + for _, node := range nodes { + n, err := newOcr2NodeFromClo(node, chainSel) + if err != nil { + return fmt.Errorf("failed to create ocr2 node from clo node: %w", err) + } + ocr2nodes = append(ocr2nodes, n) + } + _, err = configureOCR3contract(configureOCR3Request{ + cfg: cfg, + chain: registryChain, + contract: contract, + nodes: ocr2nodes, + }) + return err +} + type registerCapabilitiesRequest struct { chain deployment.Chain registry *capabilities_registry.CapabilitiesRegistry @@ -669,12 +707,13 @@ func sortedHash(p2pids [][32]byte) string { } func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsResponse, error) { - resp := ®isterDonsResponse{ + resp := registerDonsResponse{ donInfos: make(map[string]capabilities_registry.CapabilitiesRegistryDONInfo), } // track hash of sorted p2pids to don name because the registry return value does not include the don name // and we need to map it back to the don name to access the other mapping data such as the don's capabilities & nodes p2pIdsToDon := make(map[string]string) + var registeredDons = 0 for don, ocr2nodes := range req.donToOcr2Nodes { var p2pIds [][32]byte @@ -724,25 +763,47 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes return nil, fmt.Errorf("failed to confirm AddDON transaction %s for don %s: %w", tx.Hash().String(), don, err) } lggr.Debugw("registered DON", "don", don, "p2p sorted hash", p2pSortedHash, "cgs", cfgs, "wfSupported", wfSupported, "f", f) + registeredDons++ + } + lggr.Debugf("Registered all DONS %d, waiting for registry to update", registeredDons) + + // occasionally the registry does not return the expected number of DONS immediately after the txns above + // so we retry a few times. while crude, it is effective + var donInfos []capabilities_registry.CapabilitiesRegistryDONInfo + var err error + for i := 0; i < 10; i++ { + lggr.Debug("attempting to get DONS from registry", i) + donInfos, err = req.registry.GetDONs(&bind.CallOpts{}) + if len(donInfos) != registeredDons { + lggr.Debugw("expected dons not registered", "expected", registeredDons, "got", len(donInfos)) + time.Sleep(2 * time.Second) + } else { + break + } } - donInfos, err := req.registry.GetDONs(&bind.CallOpts{}) if err != nil { err = DecodeErr(kcr.CapabilitiesRegistryABI, err) return nil, fmt.Errorf("failed to call GetDONs: %w", err) } + for i, donInfo := range donInfos { donName, ok := p2pIdsToDon[sortedHash(donInfo.NodeP2PIds)] if !ok { return nil, fmt.Errorf("don not found for p2pids %s in %v", sortedHash(donInfo.NodeP2PIds), p2pIdsToDon) } + lggr.Debugw("adding don info", "don", donName, "cnt", i) resp.donInfos[donName] = donInfos[i] } - return resp, nil + lggr.Debugw("found registered DONs", "count", len(resp.donInfos)) + if len(resp.donInfos) != registeredDons { + return nil, fmt.Errorf("expected %d dons, got %d", registeredDons, len(resp.donInfos)) + } + return &resp, nil } // configureForwarder sets the config for the forwarder contract on the chain for all Dons that accept workflows // dons that don't accept workflows are not registered with the forwarder -func configureForwarder(chain deployment.Chain, fwdr *kf.KeystoneForwarder, dons []RegisteredDon) error { +func configureForwarder(lggr logger.Logger, chain deployment.Chain, fwdr *kf.KeystoneForwarder, dons []RegisteredDon) error { if fwdr == nil { return errors.New("nil forwarder contract") } @@ -761,6 +822,7 @@ func configureForwarder(chain deployment.Chain, fwdr *kf.KeystoneForwarder, dons err = DecodeErr(kf.KeystoneForwarderABI, err) return fmt.Errorf("failed to confirm SetConfig for forwarder %s: %w", fwdr.Address().String(), err) } + lggr.Debugw("configured forwarder", "forwarder", fwdr.Address().String(), "donId", dn.Info.Id, "version", ver, "f", dn.Info.F, "signers", dn.signers()) } return nil } @@ -769,7 +831,7 @@ type configureOCR3Request struct { cfg *OracleConfigSource chain deployment.Chain contract *kocr3.OCR3Capability - don RegisteredDon + nodes []*ocr2Node } type configureOCR3Response struct { ocrConfig Orc2drOracleConfig @@ -779,7 +841,7 @@ func configureOCR3contract(req configureOCR3Request) (*configureOCR3Response, er if req.contract == nil { return nil, fmt.Errorf("OCR3 contract is nil") } - nks := makeNodeKeysSlice(req.don.Nodes) + nks := makeNodeKeysSlice(req.nodes) ocrConfig, err := GenerateOCR3Config(*req.cfg, nks) if err != nil { return nil, fmt.Errorf("failed to generate OCR3 config: %w", err) diff --git a/integration-tests/deployment/keystone/types.go b/integration-tests/deployment/keystone/types.go index 439fe4ed47d..1613387343d 100644 --- a/integration-tests/deployment/keystone/types.go +++ b/integration-tests/deployment/keystone/types.go @@ -72,13 +72,20 @@ type ocr2Node struct { } func (o *ocr2Node) signerAddress() common.Address { - return common.BytesToAddress(o.Signer[:]) + // eth address is the first 20 bytes of the Signer + return common.BytesToAddress(o.Signer[:20]) } func (o *ocr2Node) toNodeKeys() NodeKeys { + var aptosOcr2KeyBundleId string + var aptosOnchainPublicKey string + if o.aptosOcr2KeyBundle != nil { + aptosOcr2KeyBundleId = o.aptosOcr2KeyBundle.BundleId + aptosOnchainPublicKey = o.aptosOcr2KeyBundle.OnchainSigningAddress + } return NodeKeys{ EthAddress: o.accountAddress, - P2PPeerID: o.p2pKeyBundle.PeerId, + P2PPeerID: strings.TrimPrefix(o.p2pKeyBundle.PeerId, "p2p_"), OCR2BundleID: o.ethOcr2KeyBundle.BundleId, OCR2OnchainPublicKey: o.ethOcr2KeyBundle.OnchainSigningAddress, OCR2OffchainPublicKey: o.ethOcr2KeyBundle.OffchainPublicKey, @@ -86,10 +93,34 @@ func (o *ocr2Node) toNodeKeys() NodeKeys { CSAPublicKey: o.csaKey, // default value of encryption public key is the CSA public key // TODO: DEVSVCS-760 - EncryptionPublicKey: o.csaKey, + EncryptionPublicKey: strings.TrimPrefix(o.csaKey, "csa_"), // TODO Aptos support. How will that be modeled in clo data? + AptosBundleID: aptosOcr2KeyBundleId, + AptosOnchainPublicKey: aptosOnchainPublicKey, } } +func newOcr2NodeFromClo(n *models.Node, registryChainSel uint64) (*ocr2Node, error) { + if n.PublicKey == nil { + return nil, errors.New("no public key") + } + // the chain configs are equivalent as far as the ocr2 config is concerned so take the first one + if len(n.ChainConfigs) == 0 { + return nil, errors.New("no chain configs") + } + // all nodes should have an evm chain config, specifically the registry chain + evmCC, err := registryChainConfig(n.ChainConfigs, chaintype.EVM, registryChainSel) + if err != nil { + return nil, fmt.Errorf("failed to get registry chain config for sel %d: %w", registryChainSel, err) + } + cfgs := map[chaintype.ChainType]*v1.ChainConfig{ + chaintype.EVM: evmCC, + } + aptosCC, exists := firstChainConfigByType(n.ChainConfigs, chaintype.Aptos) + if exists { + cfgs[chaintype.Aptos] = aptosCC + } + return newOcr2Node(n.ID, cfgs, *n.PublicKey) +} func newOcr2Node(id string, ccfgs map[chaintype.ChainType]*v1.ChainConfig, csaPubKey string) (*ocr2Node, error) { if ccfgs == nil { @@ -227,7 +258,7 @@ func mapDonsToCaps(dons []DonCapabilities) map[string][]kcr.CapabilitiesRegistry // mapDonsToNodes returns a map of don name to simplified representation of their nodes // all nodes must have evm config and ocr3 capability nodes are must also have an aptos chain config -func mapDonsToNodes(dons []DonCapabilities, excludeBootstraps bool) (map[string][]*ocr2Node, error) { +func mapDonsToNodes(dons []DonCapabilities, excludeBootstraps bool, registryChainSel uint64) (map[string][]*ocr2Node, error) { donToOcr2Nodes := make(map[string][]*ocr2Node) // get the nodes for each don from the offchain client, get ocr2 config from one of the chain configs for the node b/c // they are equivalent, and transform to ocr2node representation @@ -235,27 +266,7 @@ func mapDonsToNodes(dons []DonCapabilities, excludeBootstraps bool) (map[string] for _, don := range dons { for _, nop := range don.Nops { for _, node := range nop.Nodes { - csaPubKey := node.PublicKey - if csaPubKey == nil { - return nil, fmt.Errorf("no public key for node %s", node.ID) - } - // the chain configs are equivalent as far as the ocr2 config is concerned so take the first one - if len(node.ChainConfigs) == 0 { - return nil, fmt.Errorf("no chain configs for node %s. cannot obtain keys", node.ID) - } - // all nodes should have an evm chain config, specifically the registry chain - evmCC, exists := firstChainConfigByType(node.ChainConfigs, chaintype.EVM) - if !exists { - return nil, fmt.Errorf("no evm chain config for node %s", node.ID) - } - cfgs := map[chaintype.ChainType]*v1.ChainConfig{ - chaintype.EVM: evmCC, - } - aptosCC, exists := firstChainConfigByType(node.ChainConfigs, chaintype.Aptos) - if exists { - cfgs[chaintype.Aptos] = aptosCC - } - ocr2n, err := newOcr2Node(node.ID, cfgs, *csaPubKey) + ocr2n, err := newOcr2NodeFromClo(node, registryChainSel) if err != nil { return nil, fmt.Errorf("failed to create ocr2 node for node %s: %w", node.ID, err) } @@ -284,6 +295,21 @@ func firstChainConfigByType(ccfgs []*models.NodeChainConfig, t chaintype.ChainTy return nil, false } +func registryChainConfig(ccfgs []*models.NodeChainConfig, t chaintype.ChainType, sel uint64) (*v1.ChainConfig, error) { + chainId, err := chainsel.ChainIdFromSelector(sel) + if err != nil { + return nil, fmt.Errorf("failed to get chain id from selector %d: %w", sel, err) + } + chainIdStr := strconv.FormatUint(chainId, 10) + for _, c := range ccfgs { + //nolint:staticcheck //ignore EqualFold it broke ci for some reason (go version skew btw local and ci?) + if strings.ToLower(c.Network.ChainType.String()) == strings.ToLower(string(t)) && c.Network.ChainID == chainIdStr { + return chainConfigFromClo(c), nil + } + } + return nil, fmt.Errorf("no chain config for chain %d", chainId) +} + // RegisteredDon is a representation of a don that exists in the in the capabilities registry all with the enriched node data type RegisteredDon struct { Name string @@ -305,9 +331,9 @@ func (d RegisteredDon) signers() []common.Address { return out } -func joinInfoAndNodes(donInfos map[string]kcr.CapabilitiesRegistryDONInfo, dons []DonCapabilities) ([]RegisteredDon, error) { +func joinInfoAndNodes(donInfos map[string]kcr.CapabilitiesRegistryDONInfo, dons []DonCapabilities, registryChainSel uint64) ([]RegisteredDon, error) { // all maps should have the same keys - nodes, err := mapDonsToNodes(dons, true) + nodes, err := mapDonsToNodes(dons, true, registryChainSel) if err != nil { return nil, fmt.Errorf("failed to map dons to capabilities: %w", err) } diff --git a/integration-tests/deployment/keystone/types_test.go b/integration-tests/deployment/keystone/types_test.go index 612e98d3309..3817c46105f 100644 --- a/integration-tests/deployment/keystone/types_test.go +++ b/integration-tests/deployment/keystone/types_test.go @@ -3,11 +3,14 @@ package keystone import ( "encoding/json" "os" + "strconv" "testing" "github.com/stretchr/testify/assert" "github.com/test-go/testify/require" + chainsel "github.com/smartcontractkit/chain-selectors" + v1 "github.com/smartcontractkit/chainlink-protos/job-distributor/v1/node" "github.com/smartcontractkit/chainlink/integration-tests/deployment/clo/models" kcr "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/capabilities_registry" @@ -144,8 +147,10 @@ func Test_mapDonsToNodes(t *testing.T) { aptosSig = "b35409a8d4f9a18da55c5b2bb08a3f5f68d44442b35409a8d4f9a18da55c5b2bb08a3f5f68d44442" peerID = "p2p_12D3KooWMWUKdoAc2ruZf9f55p7NVFj7AFiPm67xjQ8BZBwkqyYv" // todo: these should be defined in common - writerCap = 3 - ocr3Cap = 2 + writerCap = 3 + ocr3Cap = 2 + registryChainSel = chainsel.ETHEREUM_TESTNET_SEPOLIA.Selector + registryChainID = strconv.FormatUint(chainsel.ETHEREUM_TESTNET_SEPOLIA.EvmChainID, 10) ) type args struct { dons []DonCapabilities @@ -172,6 +177,7 @@ func Test_mapDonsToNodes(t *testing.T) { ID: "1", Network: &models.Network{ ChainType: models.ChainTypeEvm, + ChainID: registryChainID, }, Ocr2Config: &models.NodeOCR2Config{ P2pKeyBundle: &models.NodeOCR2ConfigP2PKeyBundle{ @@ -262,6 +268,7 @@ func Test_mapDonsToNodes(t *testing.T) { ID: "1", Network: &models.Network{ ChainType: models.ChainTypeEvm, + ChainID: registryChainID, }, Ocr2Config: &models.NodeOCR2Config{ P2pKeyBundle: &models.NodeOCR2ConfigP2PKeyBundle{ @@ -296,7 +303,7 @@ func Test_mapDonsToNodes(t *testing.T) { args: args{ dons: []DonCapabilities{ { - Name: "bad chain", + Name: "ok chain", Nops: []*models.NodeOperator{ { Nodes: []*models.Node{ @@ -307,6 +314,7 @@ func Test_mapDonsToNodes(t *testing.T) { ID: "1", Network: &models.Network{ ChainType: models.ChainTypeEvm, + ChainID: registryChainID, }, Ocr2Config: &models.NodeOCR2Config{ P2pKeyBundle: &models.NodeOCR2ConfigP2PKeyBundle{ @@ -355,7 +363,7 @@ func Test_mapDonsToNodes(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := mapDonsToNodes(tt.args.dons, tt.args.excludeBootstraps) + _, err := mapDonsToNodes(tt.args.dons, tt.args.excludeBootstraps, registryChainSel) if (err != nil) != tt.wantErr { t.Errorf("mapDonsToNodes() error = %v, wantErr %v", err, tt.wantErr) return @@ -385,11 +393,11 @@ func Test_mapDonsToNodes(t *testing.T) { Nops: assetNops, Capabilities: []kcr.CapabilitiesRegistryCapability{StreamTriggerCap}, } - _, err := mapDonsToNodes([]DonCapabilities{wfDon}, false) + _, err := mapDonsToNodes([]DonCapabilities{wfDon}, false, registryChainSel) require.NoError(t, err, "failed to map wf don") - _, err = mapDonsToNodes([]DonCapabilities{cwDon}, false) + _, err = mapDonsToNodes([]DonCapabilities{cwDon}, false, registryChainSel) require.NoError(t, err, "failed to map cw don") - _, err = mapDonsToNodes([]DonCapabilities{assetDon}, false) + _, err = mapDonsToNodes([]DonCapabilities{assetDon}, false, registryChainSel) require.NoError(t, err, "failed to map asset don") }