Skip to content

Commit

Permalink
[Keystone][Deployments] Refactor RegisterDONs() (#15534)
Browse files Browse the repository at this point in the history 
1. Make it ordered.
2. Pass the F value in the request.
  • Loading branch information
@bolekk @jhweintraub
bolekk authored and jhweintraub committed Dec 6, 2024
1 parent cae509e commit 783fd12
Show file tree
Hide file tree
Showing 2 changed files with 72 additions and 41 deletions.
112 changes: 71 additions & 41 deletions deployment/keystone/deploy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"golang.org/x/exp/maps"

"github.com/smartcontractkit/ccip-owner-contracts/pkg/proposal/timelock"

"github.com/smartcontractkit/chainlink/deployment"

"google.golang.org/protobuf/proto"
Expand Down Expand Up @@ -152,6 +153,7 @@ func DeployContracts(e *deployment.Environment, chainSel uint64) (*deployment.Ch
// DonInfo is DonCapabilities, but expanded to contain node information
type DonInfo struct {
Name string
F uint8
Nodes []deployment.Node
Capabilities []kcr.CapabilitiesRegistryCapability // every capability is hosted on each node
}
Expand All @@ -169,6 +171,7 @@ func DonInfos(dons []DonCapabilities, jd deployment.OffchainClient) ([]DonInfo,
}
donInfos = append(donInfos, DonInfo{
Name: don.Name,
F: don.F,
Nodes: nodes,
Capabilities: don.Capabilities,
})
Expand Down Expand Up @@ -207,11 +210,6 @@ func GetRegistryContract(e *deployment.Environment, registryChainSel uint64, add
// ConfigureRegistry configures the registry contract with the given DONS and their capabilities
// the address book is required to contain the addresses of the deployed registry contract
func ConfigureRegistry(ctx context.Context, lggr logger.Logger, req ConfigureContractsRequest, addrBook deployment.AddressBook) (*ConfigureContractsResponse, error) {
registry, registryChain, err := GetRegistryContract(req.Env, req.RegistryChainSel, addrBook)
if err != nil {
return nil, fmt.Errorf("failed to get registry: %w", err)
}

donInfos, err := DonInfos(req.Dons, req.Env.Offchain)
if err != nil {
return nil, fmt.Errorf("failed to get don infos: %w", err)
Expand Down Expand Up @@ -271,24 +269,47 @@ func ConfigureRegistry(ctx context.Context, lggr logger.Logger, req ConfigureCon

// TODO: annotate nodes with node_operator_id in JD?

donsToRegister := []DONToRegister{}
for _, don := range req.Dons {
nodes, ok := donToNodes[don.Name]
if !ok {
return nil, fmt.Errorf("nodes not found for don %s", don.Name)
}
f := don.F
if f == 0 {
// TODO: fallback to a default value for compatibility - change to error
f = uint8(len(nodes) / 3)
lggr.Warnw("F not set for don - falling back to default", "don", don.Name, "f", f)
}
donsToRegister = append(donsToRegister, DONToRegister{
Name: don.Name,
F: f,
Nodes: nodes,
})
}

nodeIdToP2PID := map[string][32]byte{}
for nodeID, params := range nodesResp.nodeIDToParams {
nodeIdToP2PID[nodeID] = params.P2pId
}
// register DONS
donsResp, err := registerDons(lggr, registerDonsRequest{
registry: registry,
chain: registryChain,
nodeIDToParams: nodesResp.nodeIDToParams,
donToCapabilities: capabilitiesResp.DonToCapabilities,
donToNodes: donToNodes,
donsResp, err := RegisterDons(lggr, RegisterDonsRequest{
Env: req.Env,
RegistryChainSelector: req.RegistryChainSel,
NodeIDToP2PID: nodeIdToP2PID,
DonToCapabilities: capabilitiesResp.DonToCapabilities,
DonsToRegister: donsToRegister,
})
if err != nil {
return nil, fmt.Errorf("failed to register DONS: %w", err)
}
lggr.Infow("registered DONs", "dons", len(donsResp.donInfos))
lggr.Infow("registered DONs", "dons", len(donsResp.DonInfos))

return &ConfigureContractsResponse{
Changeset: &deployment.ChangesetOutput{
AddressBook: addrBook,
},
DonInfos: donsResp.donInfos,
DonInfos: donsResp.DonInfos,
}, nil
}

Expand Down Expand Up @@ -797,17 +818,23 @@ func RegisterNodes(lggr logger.Logger, req *RegisterNodesRequest) (*RegisterNode
}, nil
}

type registerDonsRequest struct {
registry *kcr.CapabilitiesRegistry
chain deployment.Chain
type DONToRegister struct {
Name string
F uint8
Nodes []deployment.Node
}

type RegisterDonsRequest struct {
Env *deployment.Environment
RegistryChainSelector uint64

nodeIDToParams map[string]kcr.CapabilitiesRegistryNodeParams
donToCapabilities map[string][]RegisteredCapability
donToNodes map[string][]deployment.Node
NodeIDToP2PID map[string][32]byte
DonToCapabilities map[string][]RegisteredCapability
DonsToRegister []DONToRegister
}

type registerDonsResponse struct {
donInfos map[string]kcr.CapabilitiesRegistryDONInfo
type RegisterDonsResponse struct {
DonInfos map[string]kcr.CapabilitiesRegistryDONInfo
}

func sortedHash(p2pids [][32]byte) string {
Expand All @@ -821,14 +848,18 @@ func sortedHash(p2pids [][32]byte) string {
return hex.EncodeToString(sha256Hash.Sum(nil))
}

func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsResponse, error) {
lggr.Infow("registering DONs...", "len", len(req.donToNodes))
func RegisterDons(lggr logger.Logger, req RegisterDonsRequest) (*RegisterDonsResponse, error) {
registry, registryChain, err := GetRegistryContract(req.Env, req.RegistryChainSelector, req.Env.ExistingAddresses)
if err != nil {
return nil, fmt.Errorf("failed to get registry: %w", err)
}
lggr.Infow("registering DONs...", "len", len(req.DonsToRegister))
// track hash of sorted p2pids to don name because the registry return value does not include the don name
// and we need to map it back to the don name to access the other mapping data such as the don's capabilities & nodes
p2pIdsToDon := make(map[string]string)
var addedDons = 0

donInfos, err := req.registry.GetDONs(&bind.CallOpts{})
donInfos, err := registry.GetDONs(&bind.CallOpts{})
if err != nil {
err = DecodeErr(kcr.CapabilitiesRegistryABI, err)
return nil, fmt.Errorf("failed to call GetDONs: %w", err)
Expand All @@ -839,30 +870,30 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes
}
lggr.Infow("fetched existing DONs...", "len", len(donInfos), "lenByNodesHash", len(existingDONs))

for don, nodes := range req.donToNodes {
for _, don := range req.DonsToRegister {
var p2pIds [][32]byte
for _, n := range nodes {
for _, n := range don.Nodes {
if n.IsBootstrap {
continue
}
params, ok := req.nodeIDToParams[n.NodeID]
p2pID, ok := req.NodeIDToP2PID[n.NodeID]
if !ok {
return nil, fmt.Errorf("node params not found for non-bootstrap node %s", n.NodeID)
}
p2pIds = append(p2pIds, params.P2pId)
p2pIds = append(p2pIds, p2pID)
}

p2pSortedHash := sortedHash(p2pIds)
p2pIdsToDon[p2pSortedHash] = don
p2pIdsToDon[p2pSortedHash] = don.Name

if _, ok := existingDONs[p2pSortedHash]; ok {
lggr.Debugw("don already exists, ignoring", "don", don, "p2p sorted hash", p2pSortedHash)
continue
}

caps, ok := req.donToCapabilities[don]
caps, ok := req.DonToCapabilities[don.Name]
if !ok {
return nil, fmt.Errorf("capabilities not found for node operator %s", don)
return nil, fmt.Errorf("capabilities not found for DON %s", don.Name)
}
wfSupported := false
var cfgs []kcr.CapabilitiesRegistryCapabilityConfiguration
Expand All @@ -882,17 +913,16 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes
})
}

f := len(p2pIds) / 3 // assuming n=3f+1. TODO should come for some config.
tx, err := req.registry.AddDON(req.chain.DeployerKey, p2pIds, cfgs, true, wfSupported, uint8(f))
tx, err := registry.AddDON(registryChain.DeployerKey, p2pIds, cfgs, true, wfSupported, don.F)
if err != nil {
err = DecodeErr(kcr.CapabilitiesRegistryABI, err)
return nil, fmt.Errorf("failed to call AddDON for don '%s' p2p2Id hash %s capability %v: %w", don, p2pSortedHash, cfgs, err)
return nil, fmt.Errorf("failed to call AddDON for don '%s' p2p2Id hash %s capability %v: %w", don.Name, p2pSortedHash, cfgs, err)
}
_, err = req.chain.Confirm(tx)
_, err = registryChain.Confirm(tx)
if err != nil {
return nil, fmt.Errorf("failed to confirm AddDON transaction %s for don %s: %w", tx.Hash().String(), don, err)
return nil, fmt.Errorf("failed to confirm AddDON transaction %s for don %s: %w", tx.Hash().String(), don.Name, err)
}
lggr.Debugw("registered DON", "don", don, "p2p sorted hash", p2pSortedHash, "cgs", cfgs, "wfSupported", wfSupported, "f", f)
lggr.Debugw("registered DON", "don", don.Name, "p2p sorted hash", p2pSortedHash, "cgs", cfgs, "wfSupported", wfSupported, "f", don.F)
addedDons++
}
lggr.Debugf("Registered all DONs (new=%d), waiting for registry to update", addedDons)
Expand All @@ -902,7 +932,7 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes
foundAll := false
for i := 0; i < 10; i++ {
lggr.Debugw("attempting to get DONs from registry", "attempt#", i)
donInfos, err = req.registry.GetDONs(&bind.CallOpts{})
donInfos, err = registry.GetDONs(&bind.CallOpts{})
if !containsAllDONs(donInfos, p2pIdsToDon) {
lggr.Debugw("some expected dons not registered yet, re-checking after a delay ...")
time.Sleep(2 * time.Second)
Expand All @@ -919,8 +949,8 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes
return nil, fmt.Errorf("did not find all desired DONS")
}

resp := registerDonsResponse{
donInfos: make(map[string]kcr.CapabilitiesRegistryDONInfo),
resp := RegisterDonsResponse{
DonInfos: make(map[string]kcr.CapabilitiesRegistryDONInfo),
}
for i, donInfo := range donInfos {
donName, ok := p2pIdsToDon[sortedHash(donInfo.NodeP2PIds)]
Expand All @@ -929,7 +959,7 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes
continue
}
lggr.Debugw("adding don info to the reponse (keyed by DON name)", "don", donName)
resp.donInfos[donName] = donInfos[i]
resp.DonInfos[donName] = donInfos[i]
}
return &resp, nil
}
Expand Down
1 change: 1 addition & 0 deletions deployment/keystone/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ func (v NOP) Validate() error {
// in is in a convenient form to handle the CLO representation of the nop data
type DonCapabilities struct {
Name string
F uint8
Nops []NOP
Capabilities []kcr.CapabilitiesRegistryCapability // every capability is hosted on each nop
}
Expand Down

0 comments on commit 783fd12

Please sign in to comment.