From 70591ff5747f0fa8e10ec670542d1dc9cf208568 Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Mon, 13 Nov 2023 13:18:02 +0200 Subject: [PATCH 1/3] chore: add multiple sending key support --- .../vrfv2plus/vrfv2plus_config/config.go | 1 + .../actions/vrfv2plus/vrfv2plus_steps.go | 2 +- integration-tests/docker/test_env/cl_node.go | 11 ++++------- integration-tests/docker/test_env/test_env.go | 14 ++++++++++---- .../docker/test_env/test_env_builder.go | 18 ++++++++++++++++++ integration-tests/load/vrfv2plus/config.go | 4 ++++ .../load/vrfv2plus/vrfv2plus_test.go | 2 ++ 7 files changed, 40 insertions(+), 12 deletions(-) diff --git a/integration-tests/actions/vrfv2plus/vrfv2plus_config/config.go b/integration-tests/actions/vrfv2plus/vrfv2plus_config/config.go index a47103a8a18..c4210ed7737 100644 --- a/integration-tests/actions/vrfv2plus/vrfv2plus_config/config.go +++ b/integration-tests/actions/vrfv2plus/vrfv2plus_config/config.go @@ -4,6 +4,7 @@ import "time" type VRFV2PlusConfig struct { ChainlinkNodeFunding float64 `envconfig:"CHAINLINK_NODE_FUNDING" default:".1"` // Amount of native currency to fund each chainlink node with + NumSendingKeys int `envconfig:"NUM_SENDING_KEYS" default:"1"` // Amount of sending keys to create on the newly created node IsNativePayment bool `envconfig:"IS_NATIVE_PAYMENT" default:"false"` // Whether to use native payment or LINK token LinkNativeFeedResponse int64 `envconfig:"LINK_NATIVE_FEED_RESPONSE" default:"1000000000000000000"` // Response of the LINK/ETH feed MinimumConfirmations uint16 `envconfig:"MINIMUM_CONFIRMATIONS" default:"3"` // Minimum number of confirmations for the VRF Coordinator diff --git a/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go b/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go index 094457c2627..4d01b4a5ff6 100644 --- a/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go +++ b/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go @@ -8,6 +8,7 @@ import ( "time" "github.com/smartcontractkit/chainlink-testing-framework/utils" + "github.com/smartcontractkit/chainlink/integration-tests/types/config/node" "github.com/smartcontractkit/chainlink/v2/core/assets" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/vrfv2plus_wrapper_load_test_consumer" @@ -22,7 +23,6 @@ import ( "github.com/smartcontractkit/chainlink/integration-tests/client" "github.com/smartcontractkit/chainlink/integration-tests/contracts" "github.com/smartcontractkit/chainlink/integration-tests/docker/test_env" - "github.com/smartcontractkit/chainlink/integration-tests/types/config/node" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/vrf_coordinator_v2_5" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/vrf_v2plus_upgraded_version" chainlinkutils "github.com/smartcontractkit/chainlink/v2/core/utils" diff --git a/integration-tests/docker/test_env/cl_node.go b/integration-tests/docker/test_env/cl_node.go index 6e74e54a4f9..c388458616e 100644 --- a/integration-tests/docker/test_env/cl_node.go +++ b/integration-tests/docker/test_env/cl_node.go @@ -22,6 +22,9 @@ import ( tc "github.com/testcontainers/testcontainers-go" tcwait "github.com/testcontainers/testcontainers-go/wait" + ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2/types" + "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper" + "github.com/smartcontractkit/chainlink-testing-framework/blockchain" "github.com/smartcontractkit/chainlink-testing-framework/docker" "github.com/smartcontractkit/chainlink-testing-framework/docker/test_env" @@ -29,8 +32,6 @@ import ( "github.com/smartcontractkit/chainlink-testing-framework/logwatch" "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" "github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2/types" - "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper" "github.com/smartcontractkit/chainlink/integration-tests/client" "github.com/smartcontractkit/chainlink/integration-tests/utils" @@ -233,11 +234,7 @@ func (n *ClNode) ChainlinkNodeAddress() (common.Address, error) { return common.HexToAddress(addr), nil } -func (n *ClNode) Fund(evmClient blockchain.EVMClient, amount *big.Float) error { - toAddress, err := n.API.PrimaryEthAddress() - if err != nil { - return err - } +func (n *ClNode) Fund(evmClient blockchain.EVMClient, amount *big.Float, toAddress string) error { toAddr := common.HexToAddress(toAddress) gasEstimates, err := evmClient.EstimateGas(ethereum.CallMsg{ To: &toAddr, diff --git a/integration-tests/docker/test_env/test_env.go b/integration-tests/docker/test_env/test_env.go index b3fc1c0cf6c..b44328b7dad 100644 --- a/integration-tests/docker/test_env/test_env.go +++ b/integration-tests/docker/test_env/test_env.go @@ -162,11 +162,17 @@ func (te *CLClusterTestEnv) StartClCluster(nodeConfig *chainlink.Config, count i // FundChainlinkNodes will fund all the provided Chainlink nodes with a set amount of native currency func (te *CLClusterTestEnv) FundChainlinkNodes(amount *big.Float) error { - for _, cl := range te.ClCluster.Nodes { - if err := cl.Fund(te.EVMClient, amount); err != nil { - return fmt.Errorf("%s, err: %w", ErrFundCLNode, err) + for i, cl := range te.ClCluster.Nodes { + keys, err := te.ClCluster.NodeAPIs()[i].MustReadETHKeys() + if err != nil { + return fmt.Errorf("failed to read ETH keys from node %s, err: %w", cl.ContainerName, err) + } + for _, key := range keys.Data { + if err := cl.Fund(te.EVMClient, amount, key.Attributes.Address); err != nil { + return fmt.Errorf("%s, err: %w", ErrFundCLNode, err) + } + time.Sleep(5 * time.Second) } - time.Sleep(5 * time.Second) } return te.EVMClient.WaitForEvents() } diff --git a/integration-tests/docker/test_env/test_env_builder.go b/integration-tests/docker/test_env/test_env_builder.go index 9f64ab64c98..a2199a94159 100644 --- a/integration-tests/docker/test_env/test_env_builder.go +++ b/integration-tests/docker/test_env/test_env_builder.go @@ -39,6 +39,7 @@ type CLTestEnvBuilder struct { secretsConfig string nonDevGethNetworks []blockchain.EVMNetwork clNodesCount int + ethKeysCount int customNodeCsaKeys []string defaultNodeCsaKeys []string l zerolog.Logger @@ -108,6 +109,11 @@ func (b *CLTestEnvBuilder) WithCLNodes(clNodesCount int) *CLTestEnvBuilder { return b } +func (b *CLTestEnvBuilder) WithNumEthKeys(num int) *CLTestEnvBuilder { + b.ethKeysCount = num + return b +} + func (b *CLTestEnvBuilder) WithForwarders() *CLTestEnvBuilder { b.hasForwarders = true return b @@ -342,6 +348,18 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) { } if b.hasGeth && b.clNodesCount > 0 && b.ETHFunds != nil { + // create the specified number of eth keys + // each node has one key by default + if b.ethKeysCount >= 1 { + for _, nodeAPI := range b.te.ClCluster.NodeAPIs() { + for i := 0; i < b.ethKeysCount-1; i++ { + _, _, err := nodeAPI.CreateTxKey("evm", b.te.EVMClient.GetChainID().String()) + if err != nil { + return nil, err + } + } + } + } b.te.ParallelTransactions(true) defer b.te.ParallelTransactions(false) if err := b.te.FundChainlinkNodes(b.ETHFunds); err != nil { diff --git a/integration-tests/load/vrfv2plus/config.go b/integration-tests/load/vrfv2plus/config.go index cba3fdcde59..061f50c813c 100644 --- a/integration-tests/load/vrfv2plus/config.go +++ b/integration-tests/load/vrfv2plus/config.go @@ -7,6 +7,7 @@ import ( "github.com/pelletier/go-toml/v2" "github.com/rs/zerolog/log" + "github.com/smartcontractkit/chainlink/integration-tests/actions/vrfv2plus/vrfv2plus_config" "github.com/smartcontractkit/chainlink/v2/core/store/models" ) @@ -46,6 +47,9 @@ type ExistingEnvConfig struct { type NewEnvConfig struct { Funding + // NumSendingKeys is the number of sending keys to create on the newly created node + // each sending key is funded by the amount specified in NodeFunds + NumSendingKeys int `toml:"num_sending_keys"` } type Common struct { diff --git a/integration-tests/load/vrfv2plus/vrfv2plus_test.go b/integration-tests/load/vrfv2plus/vrfv2plus_test.go index 625510af6f9..683dbf74108 100644 --- a/integration-tests/load/vrfv2plus/vrfv2plus_test.go +++ b/integration-tests/load/vrfv2plus/vrfv2plus_test.go @@ -53,6 +53,7 @@ func TestVRFV2PlusPerformance(t *testing.T) { l := logging.GetTestLogger(t) //todo: temporary solution with envconfig and toml config until VRF-662 is implemented vrfv2PlusConfig.MinimumConfirmations = cfg.Common.MinimumConfirmations + vrfv2PlusConfig.NumSendingKeys = cfg.NewEnvConfig.NumSendingKeys lokiConfig := wasp.NewEnvLokiConfig() lc, err := wasp.NewLokiClient(lokiConfig) @@ -173,6 +174,7 @@ func TestVRFV2PlusPerformance(t *testing.T) { WithTestLogger(t). WithGeth(). WithCLNodes(1). + WithNumEthKeys(vrfv2PlusConfig.NumSendingKeys). WithFunding(big.NewFloat(vrfv2PlusConfig.ChainlinkNodeFunding)). WithCustomCleanup( func() { From 6e3cca06eee0e2fb84098c513e8829e8880ce493 Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Mon, 13 Nov 2023 13:41:38 +0200 Subject: [PATCH 2/3] chore: create multiple sending keys in on-demand test --- .../actions/vrfv2plus/vrfv2plus_steps.go | 28 +++++++++++-------- integration-tests/smoke/vrfv2plus_test.go | 2 +- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go b/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go index 4d01b4a5ff6..29607cfe3c8 100644 --- a/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go +++ b/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go @@ -153,7 +153,7 @@ func DeployVRFV2PlusWrapperConsumers(contractDeployer contracts.ContractDeployer func CreateVRFV2PlusJob( chainlinkNode *client.ChainlinkClient, coordinatorAddress string, - nativeTokenPrimaryKeyAddress string, + sendingKeys []string, pubKeyCompressed string, chainID string, minIncomingConfirmations uint16, @@ -170,7 +170,7 @@ func CreateVRFV2PlusJob( job, err := chainlinkNode.MustCreateJob(&client.VRFV2PlusJobSpec{ Name: fmt.Sprintf("vrf-v2-plus-%s", jobUUID), CoordinatorAddress: coordinatorAddress, - FromAddresses: []string{nativeTokenPrimaryKeyAddress}, + FromAddresses: sendingKeys, EVMChainID: chainID, MinIncomingConfirmations: int(minIncomingConfirmations), PublicKey: pubKeyCompressed, @@ -313,16 +313,20 @@ func SetupVRFV2_5Environment( chainID := env.EVMClient.GetChainID() - nativeTokenPrimaryKeyAddress, err := env.ClCluster.NodeAPIs()[0].PrimaryEthAddress() + sendingKeys, err := env.ClCluster.NodeAPIs()[0].MustReadETHKeys() if err != nil { - return nil, nil, nil, fmt.Errorf("%s, err %w", ErrNodePrimaryKey, err) + return nil, nil, nil, fmt.Errorf("failed to read eth keys from node, err %w", err) + } + var sendingKeyAddresses []string + for _, key := range sendingKeys.Data { + sendingKeyAddresses = append(sendingKeyAddresses, key.Attributes.Address) } - l.Info().Msg("Creating VRFV2 Plus Job") + l.Info().Strs("sendingKeys", sendingKeyAddresses).Msg("Creating VRFV2 Plus Job") job, err := CreateVRFV2PlusJob( env.ClCluster.NodeAPIs()[0], vrfv2_5Contracts.Coordinator.Address(), - nativeTokenPrimaryKeyAddress, + sendingKeyAddresses, pubKeyCompressed, chainID.String(), vrfv2PlusConfig.MinimumConfirmations, @@ -334,13 +338,13 @@ func SetupVRFV2_5Environment( // this part is here because VRFv2 can work with only a specific key // [[EVM.KeySpecific]] // Key = '...' - addr, err := env.ClCluster.Nodes[0].API.PrimaryEthAddress() - if err != nil { - return nil, nil, nil, fmt.Errorf("%s, err %w", ErrGetPrimaryKey, err) + var configOpts []node.NodeConfigOpt + for _, sk := range sendingKeyAddresses { + configOpts = append(configOpts, node.WithVRFv2EVMEstimator(sk)) } + configOpts = append(configOpts, node.WithLogPollInterval(time.Second)) nodeConfig := node.NewConfig(env.ClCluster.Nodes[0].NodeConfig, - node.WithVRFv2EVMEstimator(addr), - node.WithLogPollInterval(1*time.Second), + configOpts..., ) l.Info().Msg("Restarting Node with new sending key PriceMax configuration and log poll period configuration") err = env.ClCluster.Nodes[0].Restart(nodeConfig) @@ -357,7 +361,7 @@ func SetupVRFV2_5Environment( data := VRFV2PlusData{ vrfv2PlusKeyData, job, - nativeTokenPrimaryKeyAddress, + sendingKeyAddresses[0], chainID, } diff --git a/integration-tests/smoke/vrfv2plus_test.go b/integration-tests/smoke/vrfv2plus_test.go index 3510a1505a7..7539d348061 100644 --- a/integration-tests/smoke/vrfv2plus_test.go +++ b/integration-tests/smoke/vrfv2plus_test.go @@ -662,7 +662,7 @@ func TestVRFv2PlusMigration(t *testing.T) { _, err = vrfv2plus.CreateVRFV2PlusJob( env.ClCluster.NodeAPIs()[0], newCoordinator.Address(), - vrfv2PlusData.PrimaryEthAddress, + []string{vrfv2PlusData.PrimaryEthAddress}, vrfv2PlusData.VRFKey.Data.ID, vrfv2PlusData.ChainID.String(), vrfv2PlusConfig.MinimumConfirmations, From 4736b724558eb2847bb1573a47beb31f6e83431a Mon Sep 17 00:00:00 2001 From: Makram Kamaleddine Date: Mon, 13 Nov 2023 15:59:15 +0200 Subject: [PATCH 3/3] chore: set log file path --- .../actions/vrfv2plus/vrfv2plus_steps.go | 12 ++++++++++-- integration-tests/types/config/node/core.go | 6 ++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go b/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go index 29607cfe3c8..edd822bf447 100644 --- a/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go +++ b/integration-tests/actions/vrfv2plus/vrfv2plus_steps.go @@ -4,9 +4,12 @@ import ( "context" "fmt" "math/big" + "os" "sync" "time" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/smartcontractkit/chainlink-testing-framework/utils" "github.com/smartcontractkit/chainlink/integration-tests/types/config/node" @@ -343,10 +346,15 @@ func SetupVRFV2_5Environment( configOpts = append(configOpts, node.WithVRFv2EVMEstimator(sk)) } configOpts = append(configOpts, node.WithLogPollInterval(time.Second)) + configOpts = append(configOpts, node.WithLogFile("../logs/vrf-test-logs")) nodeConfig := node.NewConfig(env.ClCluster.Nodes[0].NodeConfig, configOpts..., ) - l.Info().Msg("Restarting Node with new sending key PriceMax configuration and log poll period configuration") + currEx, err := os.Executable() + if err != nil { + return nil, nil, nil, fmt.Errorf("error getting executable path: %s", err.Error()) + } + l.Info().Str("currEx", currEx).Msg("Restarting Node with new sending key PriceMax configuration and log poll period configuration") err = env.ClCluster.Nodes[0].Restart(nodeConfig) if err != nil { return nil, nil, nil, fmt.Errorf("%s, err %w", ErrRestartCLNode, err) @@ -892,7 +900,7 @@ func LogRandomnessRequestedEventUpgraded( Str("Request ID", randomWordsRequestedEvent.RequestId.String()). Str("Subscription ID", randomWordsRequestedEvent.SubId.String()). Str("Sender Address", randomWordsRequestedEvent.Sender.String()). - Interface("Keyhash", randomWordsRequestedEvent.KeyHash). + Str("Keyhash", hexutil.Encode(randomWordsRequestedEvent.KeyHash[:])). Uint32("Callback Gas Limit", randomWordsRequestedEvent.CallbackGasLimit). Uint32("Number of Words", randomWordsRequestedEvent.NumWords). Uint16("Minimum Request Confirmations", randomWordsRequestedEvent.MinimumRequestConfirmations). diff --git a/integration-tests/types/config/node/core.go b/integration-tests/types/config/node/core.go index 8f7fbdc1952..a9de1cd9a2c 100644 --- a/integration-tests/types/config/node/core.go +++ b/integration-tests/types/config/node/core.go @@ -244,3 +244,9 @@ func WithLogPollInterval(interval time.Duration) NodeConfigOpt { c.EVM[0].Chain.LogPollInterval = models.MustNewDuration(interval) } } + +func WithLogFile(logFile string) NodeConfigOpt { + return func(c *chainlink.Config) { + c.Core.Log.File.Dir = utils2.Ptr(logFile) + } +}