Skip to content

Commit

Permalink
keystone deployment fixes (#14928)
Browse files Browse the repository at this point in the history 
* hacking

* wip

* wip

* working with more logging

* more logging & fix signature bug

* stage working

* linter

* address comments

* fix test
  • Loading branch information
@krehermann
krehermann authored Oct 25, 2024
1 parent 42db9fd commit ac6c949
Show file tree
Hide file tree
Showing 5 changed files with 153 additions and 71 deletions.
29 changes: 11 additions & 18 deletions integration-tests/deployment/keystone/changeset/deploy_ocr3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,36 +5,29 @@ import (

"github.com/smartcontractkit/chainlink-common/pkg/logger"
"github.com/smartcontractkit/chainlink/integration-tests/deployment"
"github.com/smartcontractkit/chainlink/integration-tests/deployment/clo/models"
kslib "github.com/smartcontractkit/chainlink/integration-tests/deployment/keystone"
)

func DeployOCR3(lggr logger.Logger, env deployment.Environment, ab deployment.AddressBook, registryChainSel uint64) (deployment.ChangesetOutput, error) {
// must have capabilities registry deployed
regAddrs, err := ab.AddressesForChain(registryChainSel)
if err != nil {
return deployment.ChangesetOutput{}, fmt.Errorf("no addresses found for chain %d: %w", registryChainSel, err)
}
found := false
for _, addr := range regAddrs {
if addr.Type == kslib.CapabilityRegistryTypeVersion.Type {
found = true
break
}
}
if !found {
return deployment.ChangesetOutput{}, fmt.Errorf("no capabilities registry found for changeset %s", "0001_deploy_registry")
}

// ocr3 only deployed on registry chain
c, ok := env.Chains[registryChainSel]
if !ok {
return deployment.ChangesetOutput{}, fmt.Errorf("chain not found in environment")
}
err = kslib.DeployOCR3(lggr, c, ab)
err := kslib.DeployOCR3(lggr, c, ab)
if err != nil {
return deployment.ChangesetOutput{}, fmt.Errorf("failed to deploy OCR3Capability: %w", err)
}

return deployment.ChangesetOutput{AddressBook: ab}, nil
}

func ConfigureOCR3Contract(lggr logger.Logger, env deployment.Environment, ab deployment.AddressBook, registryChainSel uint64, nodes []*models.Node, cfg kslib.OracleConfigSource) (deployment.ChangesetOutput, error) {

err := kslib.ConfigureOCR3ContractFromCLO(&env, registryChainSel, nodes, ab, &cfg)
if err != nil {
return deployment.ChangesetOutput{}, fmt.Errorf("failed to configure OCR3Capability: %w", err)
}

return deployment.ChangesetOutput{AddressBook: ab}, nil
}
9 changes: 1 addition & 8 deletions integration-tests/deployment/keystone/changeset/deploy_ocr3_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (

"github.com/smartcontractkit/chainlink-common/pkg/logger"
"github.com/smartcontractkit/chainlink/integration-tests/deployment"
kslb "github.com/smartcontractkit/chainlink/integration-tests/deployment/keystone"
"github.com/smartcontractkit/chainlink/integration-tests/deployment/keystone/changeset"
"github.com/smartcontractkit/chainlink/integration-tests/deployment/memory"
)
Expand All @@ -26,20 +25,14 @@ func TestDeployOCR3(t *testing.T) {
env := memory.NewMemoryEnvironment(t, lggr, zapcore.DebugLevel, cfg)

registrySel := env.AllChainSelectors()[0]
// err if no capabilities registry on chain 0
_, err := changeset.DeployOCR3(lggr, env, ab, registrySel)
require.Error(t, err)

// fake capabilities registry
err = ab.Save(registrySel, "0x0000000000000000000000000000000000000001", kslb.CapabilityRegistryTypeVersion)
require.NoError(t, err)
resp, err := changeset.DeployOCR3(lggr, env, ab, registrySel)
require.NoError(t, err)
require.NotNil(t, resp)
// OCR3 should be deployed on chain 0
addrs, err := resp.AddressBook.AddressesForChain(registrySel)
require.NoError(t, err)
require.Len(t, addrs, 2)
require.Len(t, addrs, 1)

// nothing on chain 1
require.NotEqual(t, registrySel, env.AllChainSelectors()[1])
Expand Down
84 changes: 73 additions & 11 deletions integration-tests/deployment/keystone/deploy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/ethereum/go-ethereum/rpc"

"github.com/smartcontractkit/chainlink/integration-tests/deployment"
"github.com/smartcontractkit/chainlink/integration-tests/deployment/clo/models"

"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/durationpb"
Expand Down Expand Up @@ -94,7 +95,7 @@ func ConfigureContracts(ctx context.Context, lggr logger.Logger, req ConfigureCo
}

// now we have the capability registry set up we need to configure the forwarder contracts and the OCR3 contract
dons, err := joinInfoAndNodes(cfgRegistryResp.DonInfos, req.Dons)
dons, err := joinInfoAndNodes(cfgRegistryResp.DonInfos, req.Dons, req.RegistryChainSel)
if err != nil {
return nil, fmt.Errorf("failed to assimilate registry to Dons: %w", err)
}
Expand Down Expand Up @@ -170,7 +171,7 @@ func ConfigureRegistry(ctx context.Context, lggr logger.Logger, req ConfigureCon

// all the subsequent calls to the registry are in terms of nodes
// compute the mapping of dons to their nodes for reuse in various registry calls
donToOcr2Nodes, err := mapDonsToNodes(req.Dons, true)
donToOcr2Nodes, err := mapDonsToNodes(req.Dons, true, req.RegistryChainSel)
if err != nil {
return nil, fmt.Errorf("failed to map dons to nodes: %w", err)
}
Expand Down Expand Up @@ -267,7 +268,7 @@ func ConfigureForwardContracts(env *deployment.Environment, dons []RegisteredDon
return fmt.Errorf("no forwarder contract found for chain %d", chain.Selector)
}

err := configureForwarder(chain, fwrd, dons)
err := configureForwarder(env.Logger, chain, fwrd, dons)
if err != nil {
return fmt.Errorf("failed to configure forwarder for chain selector %d: %w", chain.Selector, err)
}
Expand Down Expand Up @@ -301,14 +302,14 @@ func ConfigureOCR3Contract(env *deployment.Environment, chainSel uint64, dons []
}
contract := contracts.OCR3
if contract == nil {
return fmt.Errorf("no forwarder contract found for chain %d", chainSel)
return fmt.Errorf("no ocr3 contract found for chain %d", chainSel)
}

_, err := configureOCR3contract(configureOCR3Request{
cfg: cfg,
chain: registryChain,
contract: contract,
don: don,
nodes: don.Nodes,
})
if err != nil {
return fmt.Errorf("failed to configure OCR3 contract for don %s: %w", don.Name, err)
Expand All @@ -317,6 +318,43 @@ func ConfigureOCR3Contract(env *deployment.Environment, chainSel uint64, dons []
return nil
}

func ConfigureOCR3ContractFromCLO(env *deployment.Environment, chainSel uint64, nodes []*models.Node, addrBook deployment.AddressBook, cfg *OracleConfigSource) error {
registryChain, ok := env.Chains[chainSel]
if !ok {
return fmt.Errorf("chain %d not found in environment", chainSel)
}
contractSetsResp, err := GetContractSets(&GetContractSetsRequest{
Chains: env.Chains,
AddressBook: addrBook,
})
if err != nil {
return fmt.Errorf("failed to get contract sets: %w", err)
}
contracts, ok := contractSetsResp.ContractSets[chainSel]
if !ok {
return fmt.Errorf("failed to get contract set for chain %d", chainSel)
}
contract := contracts.OCR3
if contract == nil {
return fmt.Errorf("no ocr3 contract found for chain %d", chainSel)
}
var ocr2nodes []*ocr2Node
for _, node := range nodes {
n, err := newOcr2NodeFromClo(node, chainSel)
if err != nil {
return fmt.Errorf("failed to create ocr2 node from clo node: %w", err)
}
ocr2nodes = append(ocr2nodes, n)
}
_, err = configureOCR3contract(configureOCR3Request{
cfg: cfg,
chain: registryChain,
contract: contract,
nodes: ocr2nodes,
})
return err
}

type registerCapabilitiesRequest struct {
chain deployment.Chain
registry *capabilities_registry.CapabilitiesRegistry
Expand Down Expand Up @@ -669,12 +707,13 @@ func sortedHash(p2pids [][32]byte) string {
}

func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsResponse, error) {
resp := &registerDonsResponse{
resp := registerDonsResponse{
donInfos: make(map[string]capabilities_registry.CapabilitiesRegistryDONInfo),
}
// track hash of sorted p2pids to don name because the registry return value does not include the don name
// and we need to map it back to the don name to access the other mapping data such as the don's capabilities & nodes
p2pIdsToDon := make(map[string]string)
var registeredDons = 0

for don, ocr2nodes := range req.donToOcr2Nodes {
var p2pIds [][32]byte
Expand Down Expand Up @@ -724,25 +763,47 @@ func registerDons(lggr logger.Logger, req registerDonsRequest) (*registerDonsRes
return nil, fmt.Errorf("failed to confirm AddDON transaction %s for don %s: %w", tx.Hash().String(), don, err)
}
lggr.Debugw("registered DON", "don", don, "p2p sorted hash", p2pSortedHash, "cgs", cfgs, "wfSupported", wfSupported, "f", f)
registeredDons++
}
lggr.Debugf("Registered all DONS %d, waiting for registry to update", registeredDons)

// occasionally the registry does not return the expected number of DONS immediately after the txns above
// so we retry a few times. while crude, it is effective
var donInfos []capabilities_registry.CapabilitiesRegistryDONInfo
var err error
for i := 0; i < 10; i++ {
lggr.Debug("attempting to get DONS from registry", i)
donInfos, err = req.registry.GetDONs(&bind.CallOpts{})
if len(donInfos) != registeredDons {
lggr.Debugw("expected dons not registered", "expected", registeredDons, "got", len(donInfos))
time.Sleep(2 * time.Second)
} else {
break
}
}
donInfos, err := req.registry.GetDONs(&bind.CallOpts{})
if err != nil {
err = DecodeErr(kcr.CapabilitiesRegistryABI, err)
return nil, fmt.Errorf("failed to call GetDONs: %w", err)
}

for i, donInfo := range donInfos {
donName, ok := p2pIdsToDon[sortedHash(donInfo.NodeP2PIds)]
if !ok {
return nil, fmt.Errorf("don not found for p2pids %s in %v", sortedHash(donInfo.NodeP2PIds), p2pIdsToDon)
}
lggr.Debugw("adding don info", "don", donName, "cnt", i)
resp.donInfos[donName] = donInfos[i]
}
return resp, nil
lggr.Debugw("found registered DONs", "count", len(resp.donInfos))
if len(resp.donInfos) != registeredDons {
return nil, fmt.Errorf("expected %d dons, got %d", registeredDons, len(resp.donInfos))
}
return &resp, nil
}

// configureForwarder sets the config for the forwarder contract on the chain for all Dons that accept workflows
// dons that don't accept workflows are not registered with the forwarder
func configureForwarder(chain deployment.Chain, fwdr *kf.KeystoneForwarder, dons []RegisteredDon) error {
func configureForwarder(lggr logger.Logger, chain deployment.Chain, fwdr *kf.KeystoneForwarder, dons []RegisteredDon) error {
if fwdr == nil {
return errors.New("nil forwarder contract")
}
Expand All @@ -761,6 +822,7 @@ func configureForwarder(chain deployment.Chain, fwdr *kf.KeystoneForwarder, dons
err = DecodeErr(kf.KeystoneForwarderABI, err)
return fmt.Errorf("failed to confirm SetConfig for forwarder %s: %w", fwdr.Address().String(), err)
}
lggr.Debugw("configured forwarder", "forwarder", fwdr.Address().String(), "donId", dn.Info.Id, "version", ver, "f", dn.Info.F, "signers", dn.signers())
}
return nil
}
Expand All @@ -769,7 +831,7 @@ type configureOCR3Request struct {
cfg *OracleConfigSource
chain deployment.Chain
contract *kocr3.OCR3Capability
don RegisteredDon
nodes []*ocr2Node
}
type configureOCR3Response struct {
ocrConfig Orc2drOracleConfig
Expand All @@ -779,7 +841,7 @@ func configureOCR3contract(req configureOCR3Request) (*configureOCR3Response, er
if req.contract == nil {
return nil, fmt.Errorf("OCR3 contract is nil")
}
nks := makeNodeKeysSlice(req.don.Nodes)
nks := makeNodeKeysSlice(req.nodes)
ocrConfig, err := GenerateOCR3Config(*req.cfg, nks)
if err != nil {
return nil, fmt.Errorf("failed to generate OCR3 config: %w", err)
Expand Down
80 changes: 53 additions & 27 deletions integration-tests/deployment/keystone/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,24 +72,55 @@ type ocr2Node struct {
}

func (o *ocr2Node) signerAddress() common.Address {
return common.BytesToAddress(o.Signer[:])
// eth address is the first 20 bytes of the Signer
return common.BytesToAddress(o.Signer[:20])
}

func (o *ocr2Node) toNodeKeys() NodeKeys {
var aptosOcr2KeyBundleId string
var aptosOnchainPublicKey string
if o.aptosOcr2KeyBundle != nil {
aptosOcr2KeyBundleId = o.aptosOcr2KeyBundle.BundleId
aptosOnchainPublicKey = o.aptosOcr2KeyBundle.OnchainSigningAddress
}
return NodeKeys{
EthAddress: o.accountAddress,
P2PPeerID: o.p2pKeyBundle.PeerId,
P2PPeerID: strings.TrimPrefix(o.p2pKeyBundle.PeerId, "p2p_"),
OCR2BundleID: o.ethOcr2KeyBundle.BundleId,
OCR2OnchainPublicKey: o.ethOcr2KeyBundle.OnchainSigningAddress,
OCR2OffchainPublicKey: o.ethOcr2KeyBundle.OffchainPublicKey,
OCR2ConfigPublicKey: o.ethOcr2KeyBundle.ConfigPublicKey,
CSAPublicKey: o.csaKey,
// default value of encryption public key is the CSA public key
// TODO: DEVSVCS-760
EncryptionPublicKey: o.csaKey,
EncryptionPublicKey: strings.TrimPrefix(o.csaKey, "csa_"),
// TODO Aptos support. How will that be modeled in clo data?
AptosBundleID: aptosOcr2KeyBundleId,
AptosOnchainPublicKey: aptosOnchainPublicKey,
}
}
func newOcr2NodeFromClo(n *models.Node, registryChainSel uint64) (*ocr2Node, error) {
if n.PublicKey == nil {
return nil, errors.New("no public key")
}
// the chain configs are equivalent as far as the ocr2 config is concerned so take the first one
if len(n.ChainConfigs) == 0 {
return nil, errors.New("no chain configs")
}
// all nodes should have an evm chain config, specifically the registry chain
evmCC, err := registryChainConfig(n.ChainConfigs, chaintype.EVM, registryChainSel)
if err != nil {
return nil, fmt.Errorf("failed to get registry chain config for sel %d: %w", registryChainSel, err)
}
cfgs := map[chaintype.ChainType]*v1.ChainConfig{
chaintype.EVM: evmCC,
}
aptosCC, exists := firstChainConfigByType(n.ChainConfigs, chaintype.Aptos)
if exists {
cfgs[chaintype.Aptos] = aptosCC
}
return newOcr2Node(n.ID, cfgs, *n.PublicKey)
}

func newOcr2Node(id string, ccfgs map[chaintype.ChainType]*v1.ChainConfig, csaPubKey string) (*ocr2Node, error) {
if ccfgs == nil {
Expand Down Expand Up @@ -227,35 +258,15 @@ func mapDonsToCaps(dons []DonCapabilities) map[string][]kcr.CapabilitiesRegistry

// mapDonsToNodes returns a map of don name to simplified representation of their nodes
// all nodes must have evm config and ocr3 capability nodes are must also have an aptos chain config
func mapDonsToNodes(dons []DonCapabilities, excludeBootstraps bool) (map[string][]*ocr2Node, error) {
func mapDonsToNodes(dons []DonCapabilities, excludeBootstraps bool, registryChainSel uint64) (map[string][]*ocr2Node, error) {
donToOcr2Nodes := make(map[string][]*ocr2Node)
// get the nodes for each don from the offchain client, get ocr2 config from one of the chain configs for the node b/c
// they are equivalent, and transform to ocr2node representation

for _, don := range dons {
for _, nop := range don.Nops {
for _, node := range nop.Nodes {
csaPubKey := node.PublicKey
if csaPubKey == nil {
return nil, fmt.Errorf("no public key for node %s", node.ID)
}
// the chain configs are equivalent as far as the ocr2 config is concerned so take the first one
if len(node.ChainConfigs) == 0 {
return nil, fmt.Errorf("no chain configs for node %s. cannot obtain keys", node.ID)
}
// all nodes should have an evm chain config, specifically the registry chain
evmCC, exists := firstChainConfigByType(node.ChainConfigs, chaintype.EVM)
if !exists {
return nil, fmt.Errorf("no evm chain config for node %s", node.ID)
}
cfgs := map[chaintype.ChainType]*v1.ChainConfig{
chaintype.EVM: evmCC,
}
aptosCC, exists := firstChainConfigByType(node.ChainConfigs, chaintype.Aptos)
if exists {
cfgs[chaintype.Aptos] = aptosCC
}
ocr2n, err := newOcr2Node(node.ID, cfgs, *csaPubKey)
ocr2n, err := newOcr2NodeFromClo(node, registryChainSel)
if err != nil {
return nil, fmt.Errorf("failed to create ocr2 node for node %s: %w", node.ID, err)
}
Expand Down Expand Up @@ -284,6 +295,21 @@ func firstChainConfigByType(ccfgs []*models.NodeChainConfig, t chaintype.ChainTy
return nil, false
}

func registryChainConfig(ccfgs []*models.NodeChainConfig, t chaintype.ChainType, sel uint64) (*v1.ChainConfig, error) {
chainId, err := chainsel.ChainIdFromSelector(sel)
if err != nil {
return nil, fmt.Errorf("failed to get chain id from selector %d: %w", sel, err)
}
chainIdStr := strconv.FormatUint(chainId, 10)
for _, c := range ccfgs {
//nolint:staticcheck //ignore EqualFold it broke ci for some reason (go version skew btw local and ci?)
if strings.ToLower(c.Network.ChainType.String()) == strings.ToLower(string(t)) && c.Network.ChainID == chainIdStr {
return chainConfigFromClo(c), nil
}
}
return nil, fmt.Errorf("no chain config for chain %d", chainId)
}

// RegisteredDon is a representation of a don that exists in the in the capabilities registry all with the enriched node data
type RegisteredDon struct {
Name string
Expand All @@ -305,9 +331,9 @@ func (d RegisteredDon) signers() []common.Address {
return out
}

func joinInfoAndNodes(donInfos map[string]kcr.CapabilitiesRegistryDONInfo, dons []DonCapabilities) ([]RegisteredDon, error) {
func joinInfoAndNodes(donInfos map[string]kcr.CapabilitiesRegistryDONInfo, dons []DonCapabilities, registryChainSel uint64) ([]RegisteredDon, error) {
// all maps should have the same keys
nodes, err := mapDonsToNodes(dons, true)
nodes, err := mapDonsToNodes(dons, true, registryChainSel)
if err != nil {
return nil, fmt.Errorf("failed to map dons to capabilities: %w", err)
}
Expand Down
Loading

0 comments on commit ac6c949

Please sign in to comment.