update nodes changeset mcms (#15543)

* support mcms in ocr3 contract config changeset

* test working for ocr3 config with mcms

* migrate deployment test to setup test env

* fix test

* refactor forwarder configuration changeset for mcms

* refactor capability update changeset mcms

* plumbing update nodes

* update node with mcms

.changeset/many-crews-wave.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+#internal refactor update nodes changeset to support mcms

deployment/keystone/capability_management.go

@@ -2,69 +2,81 @@ package keystone
 
 import (
 	"fmt"
-	"strings"
 
+	"github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/timelock"
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
 	"github.com/smartcontractkit/chainlink/deployment"
 	kcr "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/capabilities_registry"
 )
 
 // AddCapabilities adds the capabilities to the registry
 // it tries to add all capabilities in one go, if that fails, it falls back to adding them one by one
-func AddCapabilities(lggr logger.Logger, registry *kcr.CapabilitiesRegistry, chain deployment.Chain, capabilities []kcr.CapabilitiesRegistryCapability) error {
+
+func AddCapabilities(lggr logger.Logger, registry *kcr.CapabilitiesRegistry, chain deployment.Chain, capabilities []kcr.CapabilitiesRegistryCapability, useMCMS bool) ([]timelock.MCMSWithTimelockProposal, error) {
 	if len(capabilities) == 0 {
-		return nil
+		return nil, nil
 	}
-	// dedup capabilities
-	var deduped []kcr.CapabilitiesRegistryCapability
-	seen := make(map[string]struct{})
-	for _, cap := range capabilities {
-		if _, ok := seen[CapabilityID(cap)]; !ok {
-			seen[CapabilityID(cap)] = struct{}{}
-			deduped = append(deduped, cap)
-		}
+	deduped, err := dedupCapabilities(registry, capabilities)
+	if err != nil {
+		return nil, fmt.Errorf("failed to dedup capabilities: %w", err)
 	}
-
-	tx, err := registry.AddCapabilities(chain.DeployerKey, deduped)
+	txOpts := chain.DeployerKey
+	if useMCMS {
+		txOpts = deployment.SimTransactOpts()
+	}
+	tx, err := registry.AddCapabilities(txOpts, deduped)
 	if err != nil {
 		err = DecodeErr(kcr.CapabilitiesRegistryABI, err)
-		// no typed errors in the abi, so we have to do string matching
-		// try to add all capabilities in one go, if that fails, fall back to 1-by-1
-		if !strings.Contains(err.Error(), "CapabilityAlreadyExists") {
-			return fmt.Errorf("failed to call AddCapabilities: %w", err)
-		}
-		lggr.Warnw("capabilities already exist, falling back to 1-by-1", "capabilities", deduped)
-		for _, cap := range deduped {
-			tx, err = registry.AddCapabilities(chain.DeployerKey, []kcr.CapabilitiesRegistryCapability{cap})
-			if err != nil {
-				err = DecodeErr(kcr.CapabilitiesRegistryABI, err)
-				if strings.Contains(err.Error(), "CapabilityAlreadyExists") {
-					lggr.Warnw("capability already exists, skipping", "capability", cap)
-					continue
-				}
-				return fmt.Errorf("failed to call AddCapabilities for capability %v: %w", cap, err)
-			}
-			// 1-by-1 tx is pending and we need to wait for it to be mined
-			_, err = chain.Confirm(tx)
-			if err != nil {
-				return fmt.Errorf("failed to confirm AddCapabilities confirm transaction %s: %w", tx.Hash().String(), err)
-			}
-			lggr.Debugw("registered capability", "capability", cap)
-
-		}
-	} else {
-		// the bulk add tx is pending and we need to wait for it to be mined
+		return nil, fmt.Errorf("failed to add capabilities: %w", err)
+	}
+	var proposals []timelock.MCMSWithTimelockProposal
+	if !useMCMS {
 		_, err = chain.Confirm(tx)
 		if err != nil {
-			return fmt.Errorf("failed to confirm AddCapabilities confirm transaction %s: %w", tx.Hash().String(), err)
+			return nil, fmt.Errorf("failed to confirm AddCapabilities confirm transaction %s: %w", tx.Hash().String(), err)
 		}
 		lggr.Info("registered capabilities", "capabilities", deduped)
+	} else {
+		// TODO
 	}
-	return nil
+	return proposals, nil
 }
 
 // CapabilityID returns a unique id for the capability
 // TODO: mv to chainlink-common? ref https://github.com/smartcontractkit/chainlink/blob/4fb06b4525f03c169c121a68defa9b13677f5f20/contracts/src/v0.8/keystone/CapabilitiesRegistry.sol#L170
 func CapabilityID(c kcr.CapabilitiesRegistryCapability) string {
 	return fmt.Sprintf("%s@%s", c.LabelledName, c.Version)
 }
+
+// dedupCapabilities deduplicates the capabilities
+// dedup capabilities with respect to the registry
+// contract reverts on adding the same capability twice and that would cause the whole transaction to revert
+// which is very bad for us for mcms
+func dedupCapabilities(registry *kcr.CapabilitiesRegistry, capabilities []kcr.CapabilitiesRegistryCapability) ([]kcr.CapabilitiesRegistryCapability, error) {
+	var out []kcr.CapabilitiesRegistryCapability
+	existing, err := registry.GetCapabilities(nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to call GetCapabilities: %w", err)
+	}
+	existingByID := make(map[[32]byte]struct{})
+	for _, cap := range existing {
+		existingByID[cap.HashedId] = struct{}{}
+	}
+	seen := make(map[string]struct{})
+	for _, candidate := range capabilities {
+		h, err := registry.GetHashedCapabilityId(nil, candidate.LabelledName, candidate.Version)
+		if err != nil {
+			return nil, fmt.Errorf("failed to call GetHashedCapabilityId: %w", err)
+		}
+		// dedup input capabilities
+		if _, exists := seen[CapabilityID(candidate)]; exists {
+			continue
+		}
+		seen[CapabilityID(candidate)] = struct{}{}
+		// dedup with respect to the registry
+		if _, exists := existingByID[h]; !exists {
+			out = append(out, candidate)
+		}
+	}
+	return out, nil
+}

deployment/keystone/changeset/deploy_ocr3_test.go

@@ -46,7 +46,6 @@ func TestDeployOCR3(t *testing.T) {
 
 func TestConfigureOCR3(t *testing.T) {
 	t.Parallel()
-	lggr := logger.Test(t)
 
 	c := kslib.OracleConfig{
 		MaxFaultyOracles:    1,
@@ -119,13 +118,9 @@ func TestConfigureOCR3(t *testing.T) {
 		assert.NotNil(t, csOut.Proposals)
 		t.Logf("got: %v", csOut.Proposals[0])
 
-		contractSetsResp, err := kslib.GetContractSets(lggr, &kslib.GetContractSetsRequest{
-			Chains:      te.Env.Chains,
-			AddressBook: te.Env.ExistingAddresses,
-		})
-		require.NoError(t, err)
+		contracts := te.ContractSets()[te.RegistrySelector]
 		var timelocks = map[uint64]*gethwrappers.RBACTimelock{
-			te.RegistrySelector: contractSetsResp.ContractSets[te.RegistrySelector].Timelock,
+			te.RegistrySelector: contracts.Timelock,
 		}
 		// now apply the changeset such that the proposal is signed and execed
 		w2 := &bytes.Buffer{}

deployment/keystone/changeset/helpers_test.go

@@ -115,6 +115,7 @@ func (te TestEnv) ContractSets() map[uint64]kslib.ContractSet {
 }
 
 // SetupTestEnv sets up a keystone test environment with the given configuration
+// TODO: make more configurable; eg many tests don't need all the nodes (like when testing a registry change)
 func SetupTestEnv(t *testing.T, c TestConfig) TestEnv {
 	require.NoError(t, c.Validate())
 	lggr := logger.Test(t)
@@ -220,15 +221,13 @@ func SetupTestEnv(t *testing.T, c TestConfig) TestEnv {
 	}
 	var allDons = []keystone.DonCapabilities{wfDon, cwDon, assetDon}
 
-	_, err = kschangeset.ConfigureInitialContractsChangeset(env, kschangeset.InitialContractsCfg{
+	csOut, err := kschangeset.ConfigureInitialContractsChangeset(env, kschangeset.InitialContractsCfg{
 		RegistryChainSel: registryChainSel,
 		Dons:             allDons,
 		OCR3Config:       &ocr3Config,
 	})
 	require.NoError(t, err)
-	// TODO: KS-rm_deploy_opt
-	//require.Nil(t, csOut.AddressBook, "no new addresses should be created in configure initial contracts")
-	//require.NoError(t, env.ExistingAddresses.Merge(csOut.AddressBook))
+	require.Nil(t, csOut.AddressBook, "no new addresses should be created in configure initial contracts")
 
 	req := &keystone.GetContractSetsRequest{
 		Chains:      env.Chains,
@@ -256,8 +255,7 @@ func SetupTestEnv(t *testing.T, c TestConfig) TestEnv {
 	validateDon(t, gotRegistry, assetNodes, assetDon)
 
 	if c.UseMCMS {
-		// TODO: mcms on all the chains, currently only on the registry chain. need to fix this for forwarders
-		// deploy, configure and xfer ownership of MCMS
+		// deploy, configure and xfer ownership of MCMS on all chains
 		timelockCfgs := make(map[uint64]commontypes.MCMSWithTimelockConfig)
 		for sel := range env.Chains {
 			t.Logf("Enabling MCMS on chain %d", sel)

deployment/keystone/changeset/internal/append_node_capabilities.go

@@ -15,6 +15,7 @@ type AppendNodeCapabilitiesRequest struct {
 	Registry *kcr.CapabilitiesRegistry
 
 	P2pToCapabilities map[p2pkey.PeerID][]kcr.CapabilitiesRegistryCapability
+	UseMCMS           bool
 }
 
 func (req *AppendNodeCapabilitiesRequest) Validate() error {
@@ -36,7 +37,7 @@ func AppendNodeCapabilitiesImpl(lggr logger.Logger, req *AppendNodeCapabilitiesR
 	for _, cap := range req.P2pToCapabilities {
 		capabilities = append(capabilities, cap...)
 	}
-	err := kslib.AddCapabilities(lggr, req.Registry, req.Chain, capabilities)
+	proposals, err := kslib.AddCapabilities(lggr, req.Registry, req.Chain, capabilities, req.UseMCMS)
 	if err != nil {
 		return nil, fmt.Errorf("failed to add capabilities: %w", err)
 	}
@@ -55,10 +56,12 @@ func AppendNodeCapabilitiesImpl(lggr logger.Logger, req *AppendNodeCapabilitiesR
 		Chain:        req.Chain,
 		Registry:     req.Registry,
 		P2pToUpdates: updatesByPeer,
+		UseMCMS:      req.UseMCMS,
 	}
 	resp, err := UpdateNodes(lggr, updateNodesReq)
 	if err != nil {
 		return nil, fmt.Errorf("failed to update nodes: %w", err)
 	}
+	resp.Proposals = append(proposals, resp.Proposals...)
 	return resp, nil
 }

deployment/keystone/changeset/internal/test/utils.go

@@ -33,6 +33,7 @@ type SetupTestRegistryRequest struct {
 	P2pToCapabilities map[p2pkey.PeerID][]kcr.CapabilitiesRegistryCapability
 	NopToNodes        map[kcr.CapabilitiesRegistryNodeOperator][]*internal.P2PSignerEnc
 	Dons              []Don
+	// TODO maybe add support for MCMS at this level
 }
 
 type SetupTestRegistryResponse struct {

deployment/keystone/changeset/internal/update_don.go

@@ -9,6 +9,7 @@ import (
 	"sort"
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/timelock"
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
 	"github.com/smartcontractkit/chainlink/deployment"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/p2pkey"
@@ -31,13 +32,16 @@ type UpdateDonRequest struct {
 
 	P2PIDs            []p2pkey.PeerID    // this is the unique identifier for the don
 	CapabilityConfigs []CapabilityConfig // if Config subfield is nil, a default config is used
+
+	UseMCMS bool
 }
 
 func (r *UpdateDonRequest) appendNodeCapabilitiesRequest() *AppendNodeCapabilitiesRequest {
 	out := &AppendNodeCapabilitiesRequest{
 		Chain:             r.Chain,
 		Registry:          r.Registry,
 		P2pToCapabilities: make(map[p2pkey.PeerID][]kcr.CapabilitiesRegistryCapability),
+		UseMCMS:           r.UseMCMS,
 	}
 	for _, p2pid := range r.P2PIDs {
 		if _, exists := out.P2pToCapabilities[p2pid]; !exists {
@@ -61,7 +65,8 @@ func (r *UpdateDonRequest) Validate() error {
 }
 
 type UpdateDonResponse struct {
-	DonInfo kcr.CapabilitiesRegistryDONInfo
+	DonInfo   kcr.CapabilitiesRegistryDONInfo
+	Proposals []timelock.MCMSWithTimelockProposal
 }
 
 func UpdateDon(lggr logger.Logger, req *UpdateDonRequest) (*UpdateDonResponse, error) {

deployment/keystone/changeset/internal/update_node_capabilities.go

@@ -15,6 +15,8 @@ type UpdateNodeCapabilitiesImplRequest struct {
 	Registry *kcr.CapabilitiesRegistry
 
 	P2pToCapabilities map[p2pkey.PeerID][]kcr.CapabilitiesRegistryCapability
+
+	UseMCMS bool
 }
 
 func (req *UpdateNodeCapabilitiesImplRequest) Validate() error {
@@ -37,7 +39,7 @@ func UpdateNodeCapabilitiesImpl(lggr logger.Logger, req *UpdateNodeCapabilitiesI
 	for _, cap := range req.P2pToCapabilities {
 		capabilities = append(capabilities, cap...)
 	}
-	err := kslib.AddCapabilities(lggr, req.Registry, req.Chain, capabilities)
+	proposals, err := kslib.AddCapabilities(lggr, req.Registry, req.Chain, capabilities, req.UseMCMS)
 	if err != nil {
 		return nil, fmt.Errorf("failed to add capabilities: %w", err)
 	}
@@ -51,10 +53,12 @@ func UpdateNodeCapabilitiesImpl(lggr logger.Logger, req *UpdateNodeCapabilitiesI
 		Chain:        req.Chain,
 		Registry:     req.Registry,
 		P2pToUpdates: p2pToUpdates,
+		UseMCMS:      req.UseMCMS,
 	}
 	resp, err := UpdateNodes(lggr, updateNodesReq)
 	if err != nil {
 		return nil, fmt.Errorf("failed to update nodes: %w", err)
 	}
+	resp.Proposals = append(proposals, resp.Proposals...)
 	return resp, nil
 }

deployment/keystone/changeset/internal/update_nodes.go

@@ -5,15 +5,21 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"math/big"
 	"sort"
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
 
+	"github.com/smartcontractkit/ccip-owner-contracts/pkg/gethwrappers"
+	"github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/mcms"
+	"github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/timelock"
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
 	kcr "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/keystone/generated/capabilities_registry"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/p2pkey"
 
 	"github.com/smartcontractkit/chainlink/deployment"
+	"github.com/smartcontractkit/chainlink/deployment/common/proposalutils"
 	kslib "github.com/smartcontractkit/chainlink/deployment/keystone"
 )
 
@@ -30,6 +36,9 @@ type UpdateNodesRequest struct {
 	Registry *kcr.CapabilitiesRegistry
 
 	P2pToUpdates map[p2pkey.PeerID]NodeUpdate
+
+	ContractSet kslib.ContractSet // contract set for the given chain
+	UseMCMS     bool
 }
 
 func (req *UpdateNodesRequest) NodeParams() ([]kcr.CapabilitiesRegistryNodeParams, error) {
@@ -80,10 +89,12 @@ func (req *UpdateNodesRequest) Validate() error {
 
 type UpdateNodesResponse struct {
 	NodeParams []kcr.CapabilitiesRegistryNodeParams
+	Proposals  []timelock.MCMSWithTimelockProposal
 }
 
 // UpdateNodes updates the nodes in the registry
-// the update sets the signer and capabilities for each node. it does not append capabilities to the existing ones
+// the update sets the signer and capabilities for each node.
+// The nodes and capabilities must already exist in the registry.
 func UpdateNodes(lggr logger.Logger, req *UpdateNodesRequest) (*UpdateNodesResponse, error) {
 	if err := req.Validate(); err != nil {
 		return nil, fmt.Errorf("failed to validate request: %w", err)
@@ -94,17 +105,54 @@ func UpdateNodes(lggr logger.Logger, req *UpdateNodesRequest) (*UpdateNodesRespo
 		err = kslib.DecodeErr(kcr.CapabilitiesRegistryABI, err)
 		return nil, fmt.Errorf("failed to make node params: %w", err)
 	}
-	tx, err := req.Registry.UpdateNodes(req.Chain.DeployerKey, params)
+	txOpts := req.Chain.DeployerKey
+	if req.UseMCMS {
+		txOpts = deployment.SimTransactOpts()
+	}
+	tx, err := req.Registry.UpdateNodes(txOpts, params)
 	if err != nil {
 		err = kslib.DecodeErr(kcr.CapabilitiesRegistryABI, err)
 		return nil, fmt.Errorf("failed to call UpdateNodes: %w", err)
 	}
 
-	_, err = req.Chain.Confirm(tx)
-	if err != nil {
-		return nil, fmt.Errorf("failed to confirm UpdateNodes confirm transaction %s: %w", tx.Hash().String(), err)
+	var proposals []timelock.MCMSWithTimelockProposal
+	if !req.UseMCMS {
+		_, err = req.Chain.Confirm(tx)
+		if err != nil {
+			return nil, fmt.Errorf("failed to confirm UpdateNodes confirm transaction %s: %w", tx.Hash().String(), err)
+		}
+	} else {
+		ops := timelock.BatchChainOperation{
+			ChainIdentifier: mcms.ChainIdentifier(req.Chain.Selector),
+			Batch: []mcms.Operation{
+				{
+					To:    req.Registry.Address(),
+					Data:  tx.Data(),
+					Value: big.NewInt(0),
+				},
+			},
+		}
+		timelocksPerChain := map[uint64]common.Address{
+			req.Chain.Selector: req.ContractSet.Timelock.Address(),
+		}
+		proposerMCMSes := map[uint64]*gethwrappers.ManyChainMultiSig{
+			req.Chain.Selector: req.ContractSet.ProposerMcm,
+		}
+
+		proposal, err := proposalutils.BuildProposalFromBatches(
+			timelocksPerChain,
+			proposerMCMSes,
+			[]timelock.BatchChainOperation{ops},
+			"proposal to set update nodes",
+			0,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("failed to build proposal: %w", err)
+		}
+		proposals = append(proposals, *proposal)
 	}
-	return &UpdateNodesResponse{NodeParams: params}, nil
+
+	return &UpdateNodesResponse{NodeParams: params, Proposals: proposals}, nil
 }
 
 // AppendCapabilities appends the capabilities to the existing capabilities of the nodes listed in p2pIds in the registry