chore: multiple sending key support

integration-tests/actions/vrfv2plus/vrfv2plus_config/config.go

@@ -4,6 +4,7 @@ import "time"
 
 type VRFV2PlusConfig struct {
 	ChainlinkNodeFunding            float64 `envconfig:"CHAINLINK_NODE_FUNDING" default:".1"`                     // Amount of native currency to fund each chainlink node with
+	NumSendingKeys                  int     `envconfig:"NUM_SENDING_KEYS" default:"1"`                            // Amount of sending keys to create on the newly created node
 	IsNativePayment                 bool    `envconfig:"IS_NATIVE_PAYMENT" default:"false"`                       // Whether to use native payment or LINK token
 	LinkNativeFeedResponse          int64   `envconfig:"LINK_NATIVE_FEED_RESPONSE" default:"1000000000000000000"` // Response of the LINK/ETH feed
 	MinimumConfirmations            uint16  `envconfig:"MINIMUM_CONFIRMATIONS" default:"3"`                       // Minimum number of confirmations for the VRF Coordinator

integration-tests/actions/vrfv2plus/vrfv2plus_steps.go

@@ -4,10 +4,14 @@ import (
 	"context"
 	"fmt"
 	"math/big"
+	"os"
 	"sync"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common/hexutil"
+
 	"github.com/smartcontractkit/chainlink-testing-framework/utils"
+	"github.com/smartcontractkit/chainlink/integration-tests/types/config/node"
 
 	"github.com/smartcontractkit/chainlink/v2/core/assets"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/vrfv2plus_wrapper_load_test_consumer"
@@ -22,7 +26,6 @@ import (
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/contracts"
 	"github.com/smartcontractkit/chainlink/integration-tests/docker/test_env"
-	"github.com/smartcontractkit/chainlink/integration-tests/types/config/node"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/vrf_coordinator_v2_5"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/vrf_v2plus_upgraded_version"
 	chainlinkutils "github.com/smartcontractkit/chainlink/v2/core/utils"
@@ -153,7 +156,7 @@ func DeployVRFV2PlusWrapperConsumers(contractDeployer contracts.ContractDeployer
 func CreateVRFV2PlusJob(
 	chainlinkNode *client.ChainlinkClient,
 	coordinatorAddress string,
-	nativeTokenPrimaryKeyAddress string,
+	sendingKeys []string,
 	pubKeyCompressed string,
 	chainID string,
 	minIncomingConfirmations uint16,
@@ -170,7 +173,7 @@ func CreateVRFV2PlusJob(
 	job, err := chainlinkNode.MustCreateJob(&client.VRFV2PlusJobSpec{
 		Name:                     fmt.Sprintf("vrf-v2-plus-%s", jobUUID),
 		CoordinatorAddress:       coordinatorAddress,
-		FromAddresses:            []string{nativeTokenPrimaryKeyAddress},
+		FromAddresses:            sendingKeys,
 		EVMChainID:               chainID,
 		MinIncomingConfirmations: int(minIncomingConfirmations),
 		PublicKey:                pubKeyCompressed,
@@ -313,16 +316,20 @@ func SetupVRFV2_5Environment(
 
 	chainID := env.EVMClient.GetChainID()
 
-	nativeTokenPrimaryKeyAddress, err := env.ClCluster.NodeAPIs()[0].PrimaryEthAddress()
+	sendingKeys, err := env.ClCluster.NodeAPIs()[0].MustReadETHKeys()
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("%s, err %w", ErrNodePrimaryKey, err)
+		return nil, nil, nil, fmt.Errorf("failed to read eth keys from node, err %w", err)
+	}
+	var sendingKeyAddresses []string
+	for _, key := range sendingKeys.Data {
+		sendingKeyAddresses = append(sendingKeyAddresses, key.Attributes.Address)
 	}
 
-	l.Info().Msg("Creating VRFV2 Plus Job")
+	l.Info().Strs("sendingKeys", sendingKeyAddresses).Msg("Creating VRFV2 Plus Job")
 	job, err := CreateVRFV2PlusJob(
 		env.ClCluster.NodeAPIs()[0],
 		vrfv2_5Contracts.Coordinator.Address(),
-		nativeTokenPrimaryKeyAddress,
+		sendingKeyAddresses,
 		pubKeyCompressed,
 		chainID.String(),
 		vrfv2PlusConfig.MinimumConfirmations,
@@ -334,15 +341,20 @@ func SetupVRFV2_5Environment(
 	// this part is here because VRFv2 can work with only a specific key
 	// [[EVM.KeySpecific]]
 	//	Key = '...'
-	addr, err := env.ClCluster.Nodes[0].API.PrimaryEthAddress()
-	if err != nil {
-		return nil, nil, nil, fmt.Errorf("%s, err %w", ErrGetPrimaryKey, err)
+	var configOpts []node.NodeConfigOpt
+	for _, sk := range sendingKeyAddresses {
+		configOpts = append(configOpts, node.WithVRFv2EVMEstimator(sk))
 	}
+	configOpts = append(configOpts, node.WithLogPollInterval(time.Second))
+	configOpts = append(configOpts, node.WithLogFile("../logs/vrf-test-logs"))
 	nodeConfig := node.NewConfig(env.ClCluster.Nodes[0].NodeConfig,
-		node.WithVRFv2EVMEstimator(addr),
-		node.WithLogPollInterval(1*time.Second),
+		configOpts...,
 	)
-	l.Info().Msg("Restarting Node with new sending key PriceMax configuration and log poll period configuration")
+	currEx, err := os.Executable()
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("error getting executable path: %s", err.Error())
+	}
+	l.Info().Str("currEx", currEx).Msg("Restarting Node with new sending key PriceMax configuration and log poll period configuration")
 	err = env.ClCluster.Nodes[0].Restart(nodeConfig)
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("%s, err %w", ErrRestartCLNode, err)
@@ -357,7 +369,7 @@ func SetupVRFV2_5Environment(
 	data := VRFV2PlusData{
 		vrfv2PlusKeyData,
 		job,
-		nativeTokenPrimaryKeyAddress,
+		sendingKeyAddresses[0],
 		chainID,
 	}
 
@@ -888,7 +900,7 @@ func LogRandomnessRequestedEventUpgraded(
 		Str("Request ID", randomWordsRequestedEvent.RequestId.String()).
 		Str("Subscription ID", randomWordsRequestedEvent.SubId.String()).
 		Str("Sender Address", randomWordsRequestedEvent.Sender.String()).
-		Interface("Keyhash", randomWordsRequestedEvent.KeyHash).
+		Str("Keyhash", hexutil.Encode(randomWordsRequestedEvent.KeyHash[:])).
 		Uint32("Callback Gas Limit", randomWordsRequestedEvent.CallbackGasLimit).
 		Uint32("Number of Words", randomWordsRequestedEvent.NumWords).
 		Uint16("Minimum Request Confirmations", randomWordsRequestedEvent.MinimumRequestConfirmations).

integration-tests/docker/test_env/cl_node.go

@@ -22,15 +22,16 @@ import (
 	tc "github.com/testcontainers/testcontainers-go"
 	tcwait "github.com/testcontainers/testcontainers-go/wait"
 
+	ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2/types"
+	"github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper"
+
 	"github.com/smartcontractkit/chainlink-testing-framework/blockchain"
 	"github.com/smartcontractkit/chainlink-testing-framework/docker"
 	"github.com/smartcontractkit/chainlink-testing-framework/docker/test_env"
 	"github.com/smartcontractkit/chainlink-testing-framework/logging"
 	"github.com/smartcontractkit/chainlink-testing-framework/logwatch"
 	"github.com/smartcontractkit/chainlink/v2/core/services/chainlink"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype"
-	ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2/types"
-	"github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper"
 
 	"github.com/smartcontractkit/chainlink/integration-tests/client"
 	"github.com/smartcontractkit/chainlink/integration-tests/utils"
@@ -233,11 +234,7 @@ func (n *ClNode) ChainlinkNodeAddress() (common.Address, error) {
 	return common.HexToAddress(addr), nil
 }
 
-func (n *ClNode) Fund(evmClient blockchain.EVMClient, amount *big.Float) error {
-	toAddress, err := n.API.PrimaryEthAddress()
-	if err != nil {
-		return err
-	}
+func (n *ClNode) Fund(evmClient blockchain.EVMClient, amount *big.Float, toAddress string) error {
 	toAddr := common.HexToAddress(toAddress)
 	gasEstimates, err := evmClient.EstimateGas(ethereum.CallMsg{
 		To: &toAddr,

integration-tests/docker/test_env/test_env.go

@@ -162,11 +162,17 @@ func (te *CLClusterTestEnv) StartClCluster(nodeConfig *chainlink.Config, count i
 
 // FundChainlinkNodes will fund all the provided Chainlink nodes with a set amount of native currency
 func (te *CLClusterTestEnv) FundChainlinkNodes(amount *big.Float) error {
-	for _, cl := range te.ClCluster.Nodes {
-		if err := cl.Fund(te.EVMClient, amount); err != nil {
-			return fmt.Errorf("%s, err: %w", ErrFundCLNode, err)
+	for i, cl := range te.ClCluster.Nodes {
+		keys, err := te.ClCluster.NodeAPIs()[i].MustReadETHKeys()
+		if err != nil {
+			return fmt.Errorf("failed to read ETH keys from node %s, err: %w", cl.ContainerName, err)
+		}
+		for _, key := range keys.Data {
+			if err := cl.Fund(te.EVMClient, amount, key.Attributes.Address); err != nil {
+				return fmt.Errorf("%s, err: %w", ErrFundCLNode, err)
+			}
+			time.Sleep(5 * time.Second)
 		}
-		time.Sleep(5 * time.Second)
 	}
 	return te.EVMClient.WaitForEvents()
 }

integration-tests/docker/test_env/test_env_builder.go

@@ -39,6 +39,7 @@ type CLTestEnvBuilder struct {
 	secretsConfig          string
 	nonDevGethNetworks     []blockchain.EVMNetwork
 	clNodesCount           int
+	ethKeysCount           int
 	customNodeCsaKeys      []string
 	defaultNodeCsaKeys     []string
 	l                      zerolog.Logger
@@ -108,6 +109,11 @@ func (b *CLTestEnvBuilder) WithCLNodes(clNodesCount int) *CLTestEnvBuilder {
 	return b
 }
 
+func (b *CLTestEnvBuilder) WithNumEthKeys(num int) *CLTestEnvBuilder {
+	b.ethKeysCount = num
+	return b
+}
+
 func (b *CLTestEnvBuilder) WithForwarders() *CLTestEnvBuilder {
 	b.hasForwarders = true
 	return b
@@ -342,6 +348,18 @@ func (b *CLTestEnvBuilder) Build() (*CLClusterTestEnv, error) {
 	}
 
 	if b.hasGeth && b.clNodesCount > 0 && b.ETHFunds != nil {
+		// create the specified number of eth keys
+		// each node has one key by default
+		if b.ethKeysCount >= 1 {
+			for _, nodeAPI := range b.te.ClCluster.NodeAPIs() {
+				for i := 0; i < b.ethKeysCount-1; i++ {
+					_, _, err := nodeAPI.CreateTxKey("evm", b.te.EVMClient.GetChainID().String())
+					if err != nil {
+						return nil, err
+					}
+				}
+			}
+		}
 		b.te.ParallelTransactions(true)
 		defer b.te.ParallelTransactions(false)
 		if err := b.te.FundChainlinkNodes(b.ETHFunds); err != nil {

integration-tests/load/vrfv2plus/config.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/pelletier/go-toml/v2"
 	"github.com/rs/zerolog/log"
+
 	"github.com/smartcontractkit/chainlink/integration-tests/actions/vrfv2plus/vrfv2plus_config"
 	"github.com/smartcontractkit/chainlink/v2/core/store/models"
 )
@@ -46,6 +47,9 @@ type ExistingEnvConfig struct {
 
 type NewEnvConfig struct {
 	Funding
+	// NumSendingKeys is the number of sending keys to create on the newly created node
+	// each sending key is funded by the amount specified in NodeFunds
+	NumSendingKeys int `toml:"num_sending_keys"`
 }
 
 type Common struct {

integration-tests/load/vrfv2plus/vrfv2plus_test.go

@@ -53,6 +53,7 @@ func TestVRFV2PlusPerformance(t *testing.T) {
 	l := logging.GetTestLogger(t)
 	//todo: temporary solution with envconfig and toml config until VRF-662 is implemented
 	vrfv2PlusConfig.MinimumConfirmations = cfg.Common.MinimumConfirmations
+	vrfv2PlusConfig.NumSendingKeys = cfg.NewEnvConfig.NumSendingKeys
 
 	lokiConfig := wasp.NewEnvLokiConfig()
 	lc, err := wasp.NewLokiClient(lokiConfig)
@@ -173,6 +174,7 @@ func TestVRFV2PlusPerformance(t *testing.T) {
 			WithTestLogger(t).
 			WithGeth().
 			WithCLNodes(1).
+			WithNumEthKeys(vrfv2PlusConfig.NumSendingKeys).
 			WithFunding(big.NewFloat(vrfv2PlusConfig.ChainlinkNodeFunding)).
 			WithCustomCleanup(
 				func() {

integration-tests/smoke/vrfv2plus_test.go

@@ -662,7 +662,7 @@ func TestVRFv2PlusMigration(t *testing.T) {
 	_, err = vrfv2plus.CreateVRFV2PlusJob(
 		env.ClCluster.NodeAPIs()[0],
 		newCoordinator.Address(),
-		vrfv2PlusData.PrimaryEthAddress,
+		[]string{vrfv2PlusData.PrimaryEthAddress},
 		vrfv2PlusData.VRFKey.Data.ID,
 		vrfv2PlusData.ChainID.String(),
 		vrfv2PlusConfig.MinimumConfirmations,

integration-tests/types/config/node/core.go

@@ -244,3 +244,9 @@ func WithLogPollInterval(interval time.Duration) NodeConfigOpt {
 		c.EVM[0].Chain.LogPollInterval = models.MustNewDuration(interval)
 	}
 }
+
+func WithLogFile(logFile string) NodeConfigOpt {
+	return func(c *chainlink.Config) {
+		c.Core.Log.File.Dir = utils2.Ptr(logFile)
+	}
+}