From 18253b4c87d2ac957845d887b1dceda4f78a505c Mon Sep 17 00:00:00 2001 From: skudasov Date: Tue, 28 Nov 2023 20:52:50 +0100 Subject: [PATCH 1/3] run OCRv1 on devspace --- charts/chainlink-cluster/Chart.yaml | 6 +- charts/chainlink-cluster/README.md | 10 +- charts/chainlink-cluster/connect.toml | 11 ++ .../dashboard/cmd/dashboard_deploy.go | 7 +- .../chainlink-cluster/dashboard/dashboard.go | 83 ++++++++++-- charts/chainlink-cluster/devspace.yaml | 14 +- charts/chainlink-cluster/requirements.lock | 6 + .../templates/chainlink-cm.yaml | 12 +- .../templates/chainlink-service.yaml | 2 +- .../templates/mockserver-service.yaml | 14 ++ .../templates/mockserver.yaml | 3 +- charts/chainlink-cluster/values-raw-helm.yaml | 43 +++++- integration-tests/k8s/connect.go | 101 ++++++++++++++ integration-tests/load/ocr/README.md | 28 ++++ integration-tests/load/ocr/config.go | 71 ++++++++++ integration-tests/load/ocr/config.toml | 20 +++ integration-tests/load/ocr/gun.go | 54 ++++++++ integration-tests/load/ocr/helper.go | 66 +++++++++ integration-tests/load/ocr/ocr_test.go | 70 ++++++++++ integration-tests/load/ocr/vu.go | 125 ++++++++++++++++++ 20 files changed, 717 insertions(+), 29 deletions(-) create mode 100644 charts/chainlink-cluster/connect.toml create mode 100644 charts/chainlink-cluster/requirements.lock create mode 100644 charts/chainlink-cluster/templates/mockserver-service.yaml create mode 100644 integration-tests/k8s/connect.go create mode 100644 integration-tests/load/ocr/README.md create mode 100644 integration-tests/load/ocr/config.go create mode 100644 integration-tests/load/ocr/config.toml create mode 100644 integration-tests/load/ocr/gun.go create mode 100644 integration-tests/load/ocr/helper.go create mode 100644 integration-tests/load/ocr/ocr_test.go create mode 100644 integration-tests/load/ocr/vu.go diff --git a/charts/chainlink-cluster/Chart.yaml b/charts/chainlink-cluster/Chart.yaml index bfea29c82ec..b9c909197f8 100644 --- a/charts/chainlink-cluster/Chart.yaml +++ b/charts/chainlink-cluster/Chart.yaml @@ -2,4 +2,8 @@ apiVersion: v1 name: chainlink-cluster description: Chainlink nodes cluster version: 0.1.3 -appVersion: '2.6.0' \ No newline at end of file +appVersion: '2.6.0' +dependencies: + - name: "killgrave" + version: "1.0.1" + repository: "https://charts.deliveryhero.io/" \ No newline at end of file diff --git a/charts/chainlink-cluster/README.md b/charts/chainlink-cluster/README.md index 18d685b8a2b..e3cec129a91 100644 --- a/charts/chainlink-cluster/README.md +++ b/charts/chainlink-cluster/README.md @@ -55,10 +55,8 @@ Destroy the cluster devspace purge ``` -If you need to run some system level tests inside k8s use `runner` profile: -``` -devspace dev -p runner -``` +## Running load tests +Check this [doc](../../integration-tests/load/ocr/README.md) If you used `devspace dev ...` always use `devspace reset pods` to switch the pods back @@ -66,8 +64,6 @@ If you used `devspace dev ...` always use `devspace reset pods` to switch the po If you need to debug CL node that is already deployed change `dev.app.container` and `dev.app.labelSelector` in [devspace.yaml](devspace.yaml) if they are not default and run: ``` devspace dev -p node -or -devspace dev -p runner ``` ## Automatic file sync @@ -85,7 +81,7 @@ helm install -f values-raw-helm.yaml cl-cluster . ``` Forward all apps (in another terminal) ``` -sudo kubefwd svc +sudo kubefwd svc -n cl-cluster ``` Then you can connect and run your tests diff --git a/charts/chainlink-cluster/connect.toml b/charts/chainlink-cluster/connect.toml new file mode 100644 index 00000000000..f866212ed07 --- /dev/null +++ b/charts/chainlink-cluster/connect.toml @@ -0,0 +1,11 @@ +namespace = "cl-cluster" +network_name = "geth" +network_chain_id = 1337 +network_private_key = "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" +network_ws_url = "ws://geth:8546" +network_http_url = "http://geth:8544" +cl_node_url_template = "http://app-node-%d:6688" +cl_node_internal_dns_record_template = "app-node-%d" +cl_node_user = "notreal@fakeemail.ch" +cl_node_password = "fj293fbBnlQ!f9vNs" +mockserver_url = "http://app-mockserver:1080" \ No newline at end of file diff --git a/charts/chainlink-cluster/dashboard/cmd/dashboard_deploy.go b/charts/chainlink-cluster/dashboard/cmd/dashboard_deploy.go index 1ea59bc9856..93619fe6148 100644 --- a/charts/chainlink-cluster/dashboard/cmd/dashboard_deploy.go +++ b/charts/chainlink-cluster/dashboard/cmd/dashboard_deploy.go @@ -16,14 +16,11 @@ func main() { if ldsn == "" { panic("DATA_SOURCE_NAME must be provided") } + os.Setenv("DATA_SOURCE_NAME", ldsn) pdsn := os.Getenv("PROMETHEUS_DATA_SOURCE_NAME") if ldsn == "" { panic("DATA_SOURCE_NAME must be provided") } - waspDsn := os.Getenv("DATA_SOURCE_NAME") - if waspDsn == "" { - panic("DATA_SOURCE_NAME must be provided, should be the same as LOKI_DATA_SOURCE_NAME") - } dbf := os.Getenv("DASHBOARD_FOLDER") if dbf == "" { panic("DASHBOARD_FOLDER must be provided") @@ -37,7 +34,7 @@ func main() { panic("GRAFANA_TOKEN must be provided") } // if you'll use this dashboard base in other projects, you can add your own opts here to extend it - db, err := dashboard.NewCLClusterDashboard(name, ldsn, pdsn, dbf, grafanaURL, grafanaToken, nil) + db, err := dashboard.NewCLClusterDashboard(6, name, ldsn, pdsn, dbf, grafanaURL, grafanaToken, nil) if err != nil { panic(err) } diff --git a/charts/chainlink-cluster/dashboard/dashboard.go b/charts/chainlink-cluster/dashboard/dashboard.go index 47faac2dea0..b29140c0405 100644 --- a/charts/chainlink-cluster/dashboard/dashboard.go +++ b/charts/chainlink-cluster/dashboard/dashboard.go @@ -192,6 +192,7 @@ const ( // CLClusterDashboard is a dashboard for a Chainlink cluster type CLClusterDashboard struct { + Nodes int Name string LokiDataSourceName string PrometheusDataSourceName string @@ -204,8 +205,9 @@ type CLClusterDashboard struct { } // NewCLClusterDashboard returns a new dashboard for a Chainlink cluster, can be used as a base for more complex plugin based dashboards -func NewCLClusterDashboard(name, ldsn, pdsn, dbf, grafanaURL, grafanaToken string, opts []dashboard.Option) (*CLClusterDashboard, error) { +func NewCLClusterDashboard(nodes int, name, ldsn, pdsn, dbf, grafanaURL, grafanaToken string, opts []dashboard.Option) (*CLClusterDashboard, error) { db := &CLClusterDashboard{ + Nodes: nodes, Name: name, Folder: dbf, LokiDataSourceName: ldsn, @@ -236,6 +238,21 @@ func (m *CLClusterDashboard) logsRowOption(name, q string) row.Option { ) } +func (m *CLClusterDashboard) logsRowOptionsForNodes(nodes int) []row.Option { + opts := make([]row.Option, 0) + for i := 1; i <= nodes; i++ { + opts = append(opts, row.WithLogs( + fmt.Sprintf("Node %d", i), + logs.DataSource(m.LokiDataSourceName), + logs.Span(12), + logs.Height("300px"), + logs.Transparent(), + logs.WithLokiTarget(fmt.Sprintf(`{namespace="${namespace}", app="app", instance="node-%d", container="node"}`, i)), + )) + } + return opts +} + // timeseriesRowOption returns a row option for a timeseries with name, axis unit, query and legend template func (m *CLClusterDashboard) timeseriesRowOption(name, axisUnit, query, legendTemplate string) row.Option { var tsq timeseries.Option @@ -372,6 +389,8 @@ func (m *CLClusterDashboard) generate() error { m.logsRowOption("Node 2", `{namespace="${namespace}", app="app", instance="node-2", container="node"}`), m.logsRowOption("Node 3", `{namespace="${namespace}", app="app", instance="node-3", container="node"}`), m.logsRowOption("Node 4", `{namespace="${namespace}", app="app", instance="node-4", container="node"}`), + m.logsRowOption("Node 5", `{namespace="${namespace}", app="app", instance="node-5", container="node"}`), + m.logsRowOption("Node 6", `{namespace="${namespace}", app="app", instance="node-6", container="node"}`), ), // HeadTracker dashboard.Row("Head tracker", @@ -416,13 +435,13 @@ func (m *CLClusterDashboard) generate() error { "Bridge JSON Parse Values", "", `bridge_json_parse_values{namespace="${namespace}"}`, - "{{ pod }}", + "{{ pod }} JobID: {{ job_id }}", ), m.timeseriesRowOption( "OCR Median Values", "", `ocr_median_values{namespace="${namespace}"}`, - "{{ pod }}", + "{{pod}} JobID: {{ job_id }}", ), ), dashboard.Row("Relay Config Poller", @@ -827,26 +846,26 @@ func (m *CLClusterDashboard) generate() error { m.timeseriesRowOption( "Pipeline Task Execution Time", "Sec", - `pipeline_task_execution_time{namespace="${namespace}"}`, - "{{pod}}", + `pipeline_task_execution_time{namespace="${namespace}"} / 1e6`, + "{{ pod }} JobID: {{ job_id }}", ), m.timeseriesRowOption( "Pipeline Run Errors", "", `pipeline_run_errors{namespace="${namespace}"}`, - "{{pod}}", + "{{ pod }} JobID: {{ job_id }}", ), m.timeseriesRowOption( "Pipeline Run Total Time to Completion", "Sec", - `pipeline_run_total_time_to_completion{namespace="${namespace}"}`, - "{{pod}}", + `pipeline_run_total_time_to_completion{namespace="${namespace}"} / 1e6`, + "{{ pod }} JobID: {{ job_id }}", ), m.timeseriesRowOption( "Pipeline Tasks Total Finished", "", `pipeline_tasks_total_finished{namespace="${namespace}"}`, - "{{pod}}", + "{{ pod }} JobID: {{ job_id }}", ), ), dashboard.Row( @@ -865,12 +884,12 @@ func (m *CLClusterDashboard) generate() error { m.timeseriesRowOption( "Pipeline Task HTTP Fetch Time", "Sec", - `pipeline_task_http_fetch_time{namespace="${namespace}"}`, + `pipeline_task_http_fetch_time{namespace="${namespace}"} / 1e6`, "{{pod}}", ), m.timeseriesRowOption( "Pipeline Task HTTP Response Body Size", - "Sec", + "Bytes", `pipeline_task_http_response_body_size{namespace="${namespace}"}`, "{{pod}}", ), @@ -920,6 +939,48 @@ func (m *CLClusterDashboard) generate() error { ), ), } + logOptsFinal := make([]row.Option, 0) + logOptsFinal = append( + logOptsFinal, + row.Collapse(), + row.WithTimeSeries( + "Log Counters", + timeseries.Span(12), + timeseries.Height("200px"), + timeseries.DataSource(m.PrometheusDataSourceName), + timeseries.WithPrometheusTarget( + `log_panic_count{namespace="${namespace}"}`, + prometheus.Legend("{{pod}} - panic"), + ), + timeseries.WithPrometheusTarget( + `log_fatal_count{namespace="${namespace}"}`, + prometheus.Legend("{{pod}} - fatal"), + ), + timeseries.WithPrometheusTarget( + `log_critical_count{namespace="${namespace}"}`, + prometheus.Legend("{{pod}} - critical"), + ), + timeseries.WithPrometheusTarget( + `log_warn_count{namespace="${namespace}"}`, + prometheus.Legend("{{pod}} - warn"), + ), + timeseries.WithPrometheusTarget( + `log_error_count{namespace="${namespace}"}`, + prometheus.Legend("{{pod}} - error"), + ), + ), + m.logsRowOption("All errors", ` + {namespace="${namespace}", app="app", container="node"} + | json + | level="error" + | line_format "{{ .instance }} {{ .level }} {{ .ts }} {{ .logger }} {{ .caller }} {{ .msg }} {{ .version }} {{ .nodeTier }} {{ .nodeName }} {{ .node }} {{ .evmChainID }} {{ .nodeOrder }} {{ .mode }} {{ .nodeState }} {{ .sentryEventID }} {{ .stacktrace }}"`), + ) + logOptsFinal = append(logOptsFinal, m.logsRowOptionsForNodes(m.Nodes)...) + logRowOpts := dashboard.Row( + "Logs", + logOptsFinal..., + ) + opts = append(opts, logRowOpts) opts = append(opts, m.extendedOpts...) builder, err := dashboard.New( "Chainlink Cluster Dashboard", diff --git a/charts/chainlink-cluster/devspace.yaml b/charts/chainlink-cluster/devspace.yaml index 688660d918e..b05b34ffe33 100644 --- a/charts/chainlink-cluster/devspace.yaml +++ b/charts/chainlink-cluster/devspace.yaml @@ -51,11 +51,17 @@ deployments: blocktime: 1 mockserver: port: 1080 + mock: + enabled: true + imposters: + path: "" + tag: "0.4.1" + port: 1080 db: stateful: false chainlink: web_port: 6688 - p2p_port: 8090 + p2p_port: 6690 nodes: - name: node-1 image: ${DEVSPACE_IMAGE} @@ -69,6 +75,12 @@ deployments: - name: node-4 image: ${DEVSPACE_IMAGE} version: latest + - name: node-5 + image: ${DEVSPACE_IMAGE} + version: latest + - name: node-6 + image: ${DEVSPACE_IMAGE} + version: latest prometheusMonitor: "true" podAnnotations: { } nodeSelector: { } diff --git a/charts/chainlink-cluster/requirements.lock b/charts/chainlink-cluster/requirements.lock new file mode 100644 index 00000000000..37ded2eb0be --- /dev/null +++ b/charts/chainlink-cluster/requirements.lock @@ -0,0 +1,6 @@ +dependencies: +- name: killgrave + repository: https://charts.deliveryhero.io/ + version: 1.0.1 +digest: sha256:a8b224810a4fd90c3a26e6eda96de8cf0ece1c58fcb314c4fe5ab9ec9e21fcbb +generated: "2023-11-28T14:38:48.410654+01:00" diff --git a/charts/chainlink-cluster/templates/chainlink-cm.yaml b/charts/chainlink-cluster/templates/chainlink-cm.yaml index 736a3322048..b33e29df4b5 100644 --- a/charts/chainlink-cluster/templates/chainlink-cm.yaml +++ b/charts/chainlink-cluster/templates/chainlink-cm.yaml @@ -26,15 +26,25 @@ data: AllowOrigins = '*' SecureCookies = false SessionTimeout = '999h0m0s' + [Feature] + FeedsManager = true + LogPoller = true + UICSAKeys = true [OCR] Enabled = true + DefaultTransactionQueueDepth = 0 [P2P] [P2P.V2] Enabled = true - ListenAddresses = ["0.0.0.0:6690"] + ListenAddresses = ['0.0.0.0:6690'] + AnnounceAddresses = ['0.0.0.0:6690'] + DeltaDial = '500ms' + DeltaReconcile = '5s' [[EVM]] ChainID = '1337' MinContractPayment = '0' + AutoCreateKey = true + FinalityDepth = 1 [[EVM.Nodes]] Name = 'node-0' WSURL = 'ws://geth:8546' diff --git a/charts/chainlink-cluster/templates/chainlink-service.yaml b/charts/chainlink-cluster/templates/chainlink-service.yaml index 24c96c909ea..7a3b70efb4f 100644 --- a/charts/chainlink-cluster/templates/chainlink-service.yaml +++ b/charts/chainlink-cluster/templates/chainlink-service.yaml @@ -12,7 +12,7 @@ spec: port: {{ $.Values.chainlink.p2p_port }} targetPort: {{ $.Values.chainlink.p2p_port }} selector: - app: {{ $.Release.Name }} + instance: {{ $cfg.name }} type: ClusterIP --- {{- end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/mockserver-service.yaml b/charts/chainlink-cluster/templates/mockserver-service.yaml new file mode 100644 index 00000000000..f8ab78a84b5 --- /dev/null +++ b/charts/chainlink-cluster/templates/mockserver-service.yaml @@ -0,0 +1,14 @@ +{{ if (hasKey .Values "mockserver") }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-mockserver +spec: + selector: + app: {{ .Release.Name }}-mockserver + ports: + - name: serviceport + port: {{ default "1080" $.Values.mockserver.port}} + targetPort: serviceport + type: ClusterIP +{{ end }} \ No newline at end of file diff --git a/charts/chainlink-cluster/templates/mockserver.yaml b/charts/chainlink-cluster/templates/mockserver.yaml index 96f9582435f..14c05d0acd5 100755 --- a/charts/chainlink-cluster/templates/mockserver.yaml +++ b/charts/chainlink-cluster/templates/mockserver.yaml @@ -1,3 +1,4 @@ +{{ if (hasKey .Values "mockserver") }} apiVersion: apps/v1 kind: Deployment metadata: @@ -5,7 +6,6 @@ metadata: labels: app: {{ .Release.Name }}-mockserver spec: - replicas: {{ .Values.replicaCount }} selector: matchLabels: app: {{ .Release.Name }}-mockserver @@ -57,4 +57,5 @@ spec: tolerations: {{ toYaml . | indent 8 }} {{- end }} +{{- end }} --- \ No newline at end of file diff --git a/charts/chainlink-cluster/values-raw-helm.yaml b/charts/chainlink-cluster/values-raw-helm.yaml index 006515f0a33..7f032562778 100644 --- a/charts/chainlink-cluster/values-raw-helm.yaml +++ b/charts/chainlink-cluster/values-raw-helm.yaml @@ -11,7 +11,7 @@ # version: stable chainlink: web_port: 6688 - p2p_port: 8090 + p2p_port: 6690 nodes: - name: node-1 image: "public.ecr.aws/chainlink/chainlink:latest" @@ -45,9 +45,13 @@ chainlink: # HTTPURL = 'http://geth:8544' # [WebServer.TLS] # HTTPSPort = 0 +# or use overridesToml to override some part of configuration +# overridesToml: | - name: node-2 - name: node-3 - name: node-4 + - name: node-5 + - name: node-6 resources: requests: cpu: 350m @@ -106,6 +110,43 @@ runner: limits: cpu: 1 memory: 512Mi +killgrave: + nameOverride: "" + labels: { } + replicaCount: 1 + hpa: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + targetMemoryUtilizationPercentage: false + resources: { } + mock: + imposters: + # mock.imposters.configmap -- The name of the configmap containing all your imposters + configmap: example-imposters + # mock.imposters.path -- The mounting path for your imposters folder + path: /imposters + schemas: + configmap: example-schemas + path: /schemas + killgrave: + tag: "0.4.1" + secure: false + + affinity: { } + tolerations: [ ] + nodeSelector: { } + ingress: + enabled: false + className: "" + hosts: [ ] + tls: [ ] + annotations: { } + service: + type: NodePort + port: 8080 + # monitoring.coreos.com/v1 PodMonitor for each node prometheusMonitor: false diff --git a/integration-tests/k8s/connect.go b/integration-tests/k8s/connect.go new file mode 100644 index 00000000000..cafc74a51aa --- /dev/null +++ b/integration-tests/k8s/connect.go @@ -0,0 +1,101 @@ +package k8s + +import ( + "fmt" + "os" + "time" + + "github.com/pelletier/go-toml/v2" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/smartcontractkit/chainlink-testing-framework/blockchain" + client2 "github.com/smartcontractkit/chainlink-testing-framework/client" + "github.com/smartcontractkit/chainlink/integration-tests/client" + "github.com/smartcontractkit/chainlink/integration-tests/contracts" +) + +const ( + DefaultConfigFilePath = "../../../charts/chainlink-cluster/connect.toml" + ErrReadConnectionConfig = "failed to read TOML environment connection config" + ErrUnmarshalConnectionConfig = "failed to unmarshal TOML environment connection config" +) + +type ConnectionVars struct { + Namespace string `toml:"namespace"` + NetworkName string `toml:"network_name"` + NetworkChainID int64 `toml:"network_chain_id"` + NetworkPrivateKey string `toml:"network_private_key"` + NetworkWSURL string `toml:"network_ws_url"` + NetworkHTTPURL string `toml:"network_http_url"` + CLNodeURLTemplate string `toml:"cl_node_url_template"` + CLNodeInternalDNSRecordTemplate string `toml:"cl_node_internal_dns_record_template"` + CLNodeUser string `toml:"cl_node_user"` + CLNodePassword string `toml:"cl_node_password"` + MockServerURL string `toml:"mockserver_url"` +} + +// ConnectRemote connects to a local environment, see charts/chainlink-cluster +func ConnectRemote(l zerolog.Logger, net *blockchain.EVMNetwork) (blockchain.EVMClient, *client2.MockserverClient, contracts.ContractDeployer, *client.ChainlinkK8sClient, []*client.ChainlinkK8sClient, error) { + cfg, err := ReadConfig() + if err != nil { + return nil, nil, nil, nil, nil, err + } + net = &blockchain.EVMNetwork{ + Name: cfg.NetworkName, + Simulated: true, + SupportsEIP1559: true, + ClientImplementation: blockchain.EthereumClientImplementation, + ChainID: 1337, + PrivateKeys: []string{ + cfg.NetworkPrivateKey, + }, + URLs: []string{cfg.NetworkWSURL}, + HTTPURLs: []string{cfg.NetworkHTTPURL}, + ChainlinkTransactionLimit: 500000, + Timeout: blockchain.JSONStrDuration{Duration: 2 * time.Minute}, + MinimumConfirmations: 1, + GasEstimationBuffer: 10000, + } + cc, err := blockchain.NewEVMClientFromNetwork(*net, l) + if err != nil { + return nil, nil, nil, nil, nil, err + } + cd, err := contracts.NewContractDeployer(cc, l) + if err != nil { + return nil, nil, nil, nil, nil, err + } + clClients := make([]*client.ChainlinkK8sClient, 0) + for i := 1; i <= 6; i++ { + c, err := client.NewChainlinkK8sClient(&client.ChainlinkConfig{ + URL: fmt.Sprintf(cfg.CLNodeURLTemplate, i), + Email: cfg.CLNodeUser, + InternalIP: fmt.Sprintf(cfg.CLNodeInternalDNSRecordTemplate, i), + Password: cfg.CLNodePassword, + }, fmt.Sprintf(cfg.CLNodeInternalDNSRecordTemplate, i), cfg.Namespace) + if err != nil { + return nil, nil, nil, nil, nil, err + } + clClients = append(clClients, c) + } + msClient := client2.NewMockserverClient(&client2.MockserverConfig{ + LocalURL: cfg.MockServerURL, + ClusterURL: cfg.MockServerURL, + }) + return cc, msClient, cd, clClients[0], clClients[1:], nil +} + +func ReadConfig() (*ConnectionVars, error) { + var cfg *ConnectionVars + var d []byte + var err error + d, err = os.ReadFile(DefaultConfigFilePath) + if err != nil { + return nil, fmt.Errorf("%s, err: %w", ErrReadConnectionConfig, err) + } + err = toml.Unmarshal(d, &cfg) + if err != nil { + return nil, fmt.Errorf("%s, err: %w", ErrUnmarshalConnectionConfig, err) + } + log.Info().Interface("Config", cfg).Msg("Connecting to environment from config") + return cfg, nil +} diff --git a/integration-tests/load/ocr/README.md b/integration-tests/load/ocr/README.md new file mode 100644 index 00000000000..20446992dc2 --- /dev/null +++ b/integration-tests/load/ocr/README.md @@ -0,0 +1,28 @@ +### OCR Load tests + +## Setup +These tests can connect to any cluster create with [chainlink-cluster](../../../charts/chainlink-cluster/README.md) + +Create your cluster +``` +kubectl create ns my-cluster +devspace use namespace my-cluster +devspace deploy +sudo kubefwd svc -n my-cluster +``` + +Change environment connection configuration [here](connection.toml) + +If you haven't changed anything in [devspace.yaml](../../../charts/chainlink-cluster/devspace.yaml) then default connection configuration will work + +## Usage + +``` +export LOKI_TOKEN=... +export LOKI_URL=... + +go test -v -run TestOCRLoad +go test -v -run TestOCRVolume +``` + +Check test configuration [here](config.toml) \ No newline at end of file diff --git a/integration-tests/load/ocr/config.go b/integration-tests/load/ocr/config.go new file mode 100644 index 00000000000..2b18af4af16 --- /dev/null +++ b/integration-tests/load/ocr/config.go @@ -0,0 +1,71 @@ +package ocr + +import ( + "encoding/base64" + "fmt" + "os" + + "github.com/pelletier/go-toml/v2" + "github.com/rs/zerolog/log" + "github.com/smartcontractkit/chainlink/v2/core/store/models" +) + +const ( + DefaultConfigFilename = "config.toml" + ErrReadPerfConfig = "failed to read TOML config for performance tests" + ErrUnmarshalPerfConfig = "failed to unmarshal TOML config for performance tests" +) + +type PerformanceConfig struct { + Load *Load `toml:"Load"` + Volume *Volume `toml:"Volume"` + Common *Common `toml:"Common"` +} + +type Common struct { + ETHFunds int `toml:"eth_funds"` +} + +type Load struct { + TestDuration *models.Duration `toml:"test_duration"` + Rate int64 `toml:"rate"` + RateLimitUnitDuration *models.Duration `toml:"rate_limit_unit_duration"` + VerificationInterval *models.Duration `toml:"verification_interval"` + VerificationTimeout *models.Duration `toml:"verification_timeout"` + EAChangeInterval *models.Duration `toml:"ea_change_interval"` +} + +type Volume struct { + TestDuration *models.Duration `toml:"test_duration"` + Rate int64 `toml:"rate"` + VURequestsPerUnit int `toml:"vu_requests_per_unit"` + RateLimitUnitDuration *models.Duration `toml:"rate_limit_unit_duration"` + VerificationInterval *models.Duration `toml:"verification_interval"` + VerificationTimeout *models.Duration `toml:"verification_timeout"` + EAChangeInterval *models.Duration `toml:"ea_change_interval"` +} + +func ReadConfig() (*PerformanceConfig, error) { + var cfg *PerformanceConfig + rawConfig := os.Getenv("CONFIG") + var d []byte + var err error + if rawConfig == "" { + d, err = os.ReadFile(DefaultConfigFilename) + if err != nil { + return nil, fmt.Errorf("%s, err: %w", ErrReadPerfConfig, err) + } + } else { + d, err = base64.StdEncoding.DecodeString(rawConfig) + if err != nil { + return nil, fmt.Errorf("%s, err: %w", ErrReadPerfConfig, err) + } + } + err = toml.Unmarshal(d, &cfg) + if err != nil { + return nil, fmt.Errorf("%s, err: %w", ErrUnmarshalPerfConfig, err) + } + + log.Debug().Interface("Config", cfg).Msg("Parsed config") + return cfg, nil +} diff --git a/integration-tests/load/ocr/config.toml b/integration-tests/load/ocr/config.toml new file mode 100644 index 00000000000..df8364b3ee4 --- /dev/null +++ b/integration-tests/load/ocr/config.toml @@ -0,0 +1,20 @@ +[Load] +test_duration = "3m" +rate_limit_unit_duration = "1m" +rate = 3 +verification_interval = "5s" +verification_timeout = "3m" +ea_change_interval = "5s" + +[Volume] +test_duration = "3m" +rate_limit_unit_duration = "1m" +vu_requests_per_unit = 10 +rate = 1 +verification_interval = "5s" +verification_timeout = "3m" + +ea_change_interval = "5s" + +[Common] +eth_funds = 3 \ No newline at end of file diff --git a/integration-tests/load/ocr/gun.go b/integration-tests/load/ocr/gun.go new file mode 100644 index 00000000000..b205a79ed76 --- /dev/null +++ b/integration-tests/load/ocr/gun.go @@ -0,0 +1,54 @@ +package ocr + +import ( + "context" + "sync/atomic" + "time" + + "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-testing-framework/blockchain" + "github.com/smartcontractkit/chainlink/integration-tests/contracts" + + "github.com/smartcontractkit/wasp" +) + +// Gun is a gun for the OCR load test +// it triggers new rounds for provided feed(aggregator) contract +type Gun struct { + roundNum atomic.Int64 + ocrInstances []contracts.OffchainAggregator + cc blockchain.EVMClient + l zerolog.Logger +} + +func NewGun(l zerolog.Logger, cc blockchain.EVMClient, ocrInstances []contracts.OffchainAggregator) *Gun { + return &Gun{ + l: l, + cc: cc, + ocrInstances: ocrInstances, + } +} + +func (m *Gun) Call(_ *wasp.Generator) *wasp.CallResult { + m.roundNum.Add(1) + requestedRound := m.roundNum.Load() + m.l.Info(). + Int64("RoundNum", requestedRound). + Str("FeedID", m.ocrInstances[0].Address()). + Msg("starting new round") + err := m.ocrInstances[0].RequestNewRound() + if err != nil { + return &wasp.CallResult{Error: err.Error(), Failed: true} + } + for { + time.Sleep(5 * time.Second) + lr, err := m.ocrInstances[0].GetLatestRound(context.Background()) + if err != nil { + return &wasp.CallResult{Error: err.Error(), Failed: true} + } + m.l.Info().Interface("LatestRound", lr).Msg("latest round") + if lr.RoundId.Int64() >= requestedRound { + return &wasp.CallResult{} + } + } +} diff --git a/integration-tests/load/ocr/helper.go b/integration-tests/load/ocr/helper.go new file mode 100644 index 00000000000..be324f4d1b2 --- /dev/null +++ b/integration-tests/load/ocr/helper.go @@ -0,0 +1,66 @@ +package ocr + +import ( + "math/big" + "math/rand" + "time" + + "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-testing-framework/blockchain" + client2 "github.com/smartcontractkit/chainlink-testing-framework/client" + "github.com/smartcontractkit/chainlink/integration-tests/actions" + "github.com/smartcontractkit/chainlink/integration-tests/client" + "github.com/smartcontractkit/chainlink/integration-tests/contracts" +) + +func SetupCluster( + cc blockchain.EVMClient, + cd contracts.ContractDeployer, + workerNodes []*client.ChainlinkK8sClient, +) (contracts.LinkToken, error) { + err := actions.FundChainlinkNodes(workerNodes, cc, big.NewFloat(3)) + if err != nil { + return nil, err + } + lt, err := cd.DeployLinkTokenContract() + if err != nil { + return nil, err + } + return lt, nil +} + +func SetupFeed( + cc blockchain.EVMClient, + msClient *client2.MockserverClient, + cd contracts.ContractDeployer, + bootstrapNode *client.ChainlinkK8sClient, + workerNodes []*client.ChainlinkK8sClient, + lt contracts.LinkToken, +) ([]contracts.OffchainAggregator, error) { + ocrInstances, err := actions.DeployOCRContracts(1, lt, cd, workerNodes, cc) + if err != nil { + return nil, err + } + err = actions.CreateOCRJobs(ocrInstances, bootstrapNode, workerNodes, 5, msClient, cc.GetChainID().String()) + if err != nil { + return nil, err + } + return ocrInstances, nil +} + +func SimulateEAActivity( + l zerolog.Logger, + eaChangeInterval time.Duration, + ocrInstances []contracts.OffchainAggregator, + workerNodes []*client.ChainlinkK8sClient, + msClient *client2.MockserverClient, +) { + go func() { + for { + time.Sleep(eaChangeInterval) + if err := actions.SetAllAdapterResponsesToTheSameValue(rand.Intn(1000), ocrInstances, workerNodes, msClient); err != nil { + l.Error().Err(err).Msg("failed to update mockserver responses") + } + } + }() +} diff --git a/integration-tests/load/ocr/ocr_test.go b/integration-tests/load/ocr/ocr_test.go new file mode 100644 index 00000000000..a05f3ad16a2 --- /dev/null +++ b/integration-tests/load/ocr/ocr_test.go @@ -0,0 +1,70 @@ +package ocr + +import ( + "testing" + + "github.com/smartcontractkit/chainlink/integration-tests/k8s" + "github.com/smartcontractkit/wasp" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-testing-framework/logging" +) + +var ( + CommonTestLabels = map[string]string{ + "branch": "ocr_healthcheck_local", + "commit": "ocr_healthcheck_local", + } +) + +func TestOCRPerformance(t *testing.T) { + l := logging.GetTestLogger(t) + cc, msClient, cd, bootstrapNode, workerNodes, err := k8s.ConnectRemote(l, nil) + require.NoError(t, err) + lt, err := SetupCluster(cc, cd, workerNodes) + require.NoError(t, err) + ocrInstances, err := SetupFeed(cc, msClient, cd, bootstrapNode, workerNodes, lt) + require.NoError(t, err) + cfg, err := ReadConfig() + require.NoError(t, err) + SimulateEAActivity(l, cfg.Load.EAChangeInterval.Duration(), ocrInstances, workerNodes, msClient) + + p := wasp.NewProfile() + p.Add(wasp.NewGenerator(&wasp.Config{ + T: t, + GenName: "ocr", + LoadType: wasp.RPS, + CallTimeout: cfg.Load.VerificationTimeout.Duration(), + RateLimitUnitDuration: cfg.Load.RateLimitUnitDuration.Duration(), + Schedule: wasp.Plain(cfg.Load.Rate, cfg.Load.TestDuration.Duration()), + Gun: NewGun(l, cc, ocrInstances), + Labels: CommonTestLabels, + LokiConfig: wasp.NewEnvLokiConfig(), + })) + _, err = p.Run(true) + require.NoError(t, err) +} + +func TestOCRCapacity(t *testing.T) { + l := logging.GetTestLogger(t) + cc, msClient, cd, bootstrapNode, workerNodes, err := k8s.ConnectRemote(l, nil) + require.NoError(t, err) + lt, err := SetupCluster(cc, cd, workerNodes) + require.NoError(t, err) + cfg, err := ReadConfig() + require.NoError(t, err) + + p := wasp.NewProfile() + p.Add(wasp.NewGenerator(&wasp.Config{ + T: t, + GenName: "ocr", + LoadType: wasp.VU, + CallTimeout: cfg.Volume.VerificationTimeout.Duration(), + Schedule: wasp.Plain(cfg.Volume.Rate, cfg.Volume.TestDuration.Duration()), + VU: NewVU(l, cfg.Volume.VURequestsPerUnit, cfg.Volume.RateLimitUnitDuration.Duration(), cc, lt, cd, bootstrapNode, workerNodes, msClient), + Labels: CommonTestLabels, + LokiConfig: wasp.NewEnvLokiConfig(), + })) + _, err = p.Run(true) + require.NoError(t, err) +} diff --git a/integration-tests/load/ocr/vu.go b/integration-tests/load/ocr/vu.go new file mode 100644 index 00000000000..c8dcdcd3948 --- /dev/null +++ b/integration-tests/load/ocr/vu.go @@ -0,0 +1,125 @@ +package ocr + +import ( + "context" + "sync/atomic" + "time" + + "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-testing-framework/blockchain" + client2 "github.com/smartcontractkit/chainlink-testing-framework/client" + "github.com/smartcontractkit/chainlink/integration-tests/actions" + "github.com/smartcontractkit/chainlink/integration-tests/client" + "github.com/smartcontractkit/chainlink/integration-tests/contracts" + "github.com/smartcontractkit/wasp" + "go.uber.org/ratelimit" +) + +// VU is a virtual user for the OCR load test +// it creates a feed and triggers new rounds +type VU struct { + rl ratelimit.Limiter + rate int + rateUnit time.Duration + roundNum atomic.Int64 + cc blockchain.EVMClient + lt contracts.LinkToken + cd contracts.ContractDeployer + bootstrapNode *client.ChainlinkK8sClient + workerNodes []*client.ChainlinkK8sClient + msClient *client2.MockserverClient + l zerolog.Logger + ocrInstances []contracts.OffchainAggregator + stop chan struct{} +} + +func NewVU( + l zerolog.Logger, + rate int, + rateUnit time.Duration, + cc blockchain.EVMClient, + lt contracts.LinkToken, + cd contracts.ContractDeployer, + bootstrapNode *client.ChainlinkK8sClient, + workerNodes []*client.ChainlinkK8sClient, + msClient *client2.MockserverClient, +) *VU { + return &VU{ + rl: ratelimit.New(rate, ratelimit.Per(rateUnit)), + rate: rate, + rateUnit: rateUnit, + l: l, + cc: cc, + lt: lt, + cd: cd, + msClient: msClient, + bootstrapNode: bootstrapNode, + workerNodes: workerNodes, + } +} + +func (m *VU) Clone(_ *wasp.Generator) wasp.VirtualUser { + return &VU{ + stop: make(chan struct{}, 1), + rl: ratelimit.New(m.rate, ratelimit.Per(m.rateUnit)), + rate: m.rate, + rateUnit: m.rateUnit, + l: m.l, + cc: m.cc, + lt: m.lt, + cd: m.cd, + msClient: m.msClient, + bootstrapNode: m.bootstrapNode, + workerNodes: m.workerNodes, + } +} + +func (m *VU) Setup(_ *wasp.Generator) error { + ocrInstances, err := actions.DeployOCRContracts(1, m.lt, m.cd, m.workerNodes, m.cc) + if err != nil { + return err + } + err = actions.CreateOCRJobs(ocrInstances, m.bootstrapNode, m.workerNodes, 5, m.msClient, m.cc.GetChainID().String()) + if err != nil { + return err + } + m.ocrInstances = ocrInstances + return nil +} + +func (m *VU) Teardown(_ *wasp.Generator) error { + return nil +} + +func (m *VU) Call(l *wasp.Generator) { + m.rl.Take() + m.roundNum.Add(1) + requestedRound := m.roundNum.Load() + m.l.Info(). + Int64("RoundNum", requestedRound). + Str("FeedID", m.ocrInstances[0].Address()). + Msg("starting new round") + err := m.ocrInstances[0].RequestNewRound() + if err != nil { + l.ResponsesChan <- &wasp.CallResult{Error: err.Error(), Failed: true} + } + for { + time.Sleep(5 * time.Second) + lr, err := m.ocrInstances[0].GetLatestRound(context.Background()) + if err != nil { + l.ResponsesChan <- &wasp.CallResult{Error: err.Error(), Failed: true} + } + m.l.Info().Interface("LatestRound", lr).Msg("latest round") + if lr.RoundId.Int64() >= requestedRound { + l.ResponsesChan <- &wasp.CallResult{} + } + } +} + +func (m *VU) Stop(_ *wasp.Generator) { + m.stop <- struct{}{} +} + +func (m *VU) StopChan() chan struct{} { + return m.stop +} From 4be02e8e5b07cbbe13aa70e8fdb83c0981fa006e Mon Sep 17 00:00:00 2001 From: skudasov Date: Wed, 29 Nov 2023 23:00:08 +0100 Subject: [PATCH 2/3] fix go.mod --- integration-tests/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 4cf6a502c49..bb75d78af1a 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -36,6 +36,7 @@ require ( github.com/testcontainers/testcontainers-go v0.23.0 github.com/umbracle/ethgo v0.1.3 go.dedis.ch/kyber/v3 v3.1.0 + go.uber.org/ratelimit v0.2.0 go.uber.org/zap v1.26.0 golang.org/x/sync v0.5.0 gopkg.in/guregu/null.v4 v4.0.0 @@ -451,7 +452,6 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/ratelimit v0.2.0 // indirect golang.org/x/arch v0.4.0 // indirect golang.org/x/crypto v0.15.0 // indirect golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect From 472e6eee2984d27d47115e641c5de2d37a2bc41e Mon Sep 17 00:00:00 2001 From: skudasov Date: Wed, 29 Nov 2023 23:18:05 +0100 Subject: [PATCH 3/3] fix goimports --- integration-tests/k8s/connect.go | 5 +++-- integration-tests/load/ocr/config.go | 1 + integration-tests/load/ocr/gun.go | 1 + integration-tests/load/ocr/helper.go | 2 ++ integration-tests/load/ocr/ocr_test.go | 5 +++-- integration-tests/load/ocr/vu.go | 7 +++++-- 6 files changed, 15 insertions(+), 6 deletions(-) diff --git a/integration-tests/k8s/connect.go b/integration-tests/k8s/connect.go index cafc74a51aa..e927761fe2e 100644 --- a/integration-tests/k8s/connect.go +++ b/integration-tests/k8s/connect.go @@ -8,6 +8,7 @@ import ( "github.com/pelletier/go-toml/v2" "github.com/rs/zerolog" "github.com/rs/zerolog/log" + "github.com/smartcontractkit/chainlink-testing-framework/blockchain" client2 "github.com/smartcontractkit/chainlink-testing-framework/client" "github.com/smartcontractkit/chainlink/integration-tests/client" @@ -35,12 +36,12 @@ type ConnectionVars struct { } // ConnectRemote connects to a local environment, see charts/chainlink-cluster -func ConnectRemote(l zerolog.Logger, net *blockchain.EVMNetwork) (blockchain.EVMClient, *client2.MockserverClient, contracts.ContractDeployer, *client.ChainlinkK8sClient, []*client.ChainlinkK8sClient, error) { +func ConnectRemote(l zerolog.Logger) (blockchain.EVMClient, *client2.MockserverClient, contracts.ContractDeployer, *client.ChainlinkK8sClient, []*client.ChainlinkK8sClient, error) { cfg, err := ReadConfig() if err != nil { return nil, nil, nil, nil, nil, err } - net = &blockchain.EVMNetwork{ + net := &blockchain.EVMNetwork{ Name: cfg.NetworkName, Simulated: true, SupportsEIP1559: true, diff --git a/integration-tests/load/ocr/config.go b/integration-tests/load/ocr/config.go index 2b18af4af16..2991df3774a 100644 --- a/integration-tests/load/ocr/config.go +++ b/integration-tests/load/ocr/config.go @@ -7,6 +7,7 @@ import ( "github.com/pelletier/go-toml/v2" "github.com/rs/zerolog/log" + "github.com/smartcontractkit/chainlink/v2/core/store/models" ) diff --git a/integration-tests/load/ocr/gun.go b/integration-tests/load/ocr/gun.go index b205a79ed76..a2eb1ff2200 100644 --- a/integration-tests/load/ocr/gun.go +++ b/integration-tests/load/ocr/gun.go @@ -6,6 +6,7 @@ import ( "time" "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-testing-framework/blockchain" "github.com/smartcontractkit/chainlink/integration-tests/contracts" diff --git a/integration-tests/load/ocr/helper.go b/integration-tests/load/ocr/helper.go index be324f4d1b2..c35dc384d17 100644 --- a/integration-tests/load/ocr/helper.go +++ b/integration-tests/load/ocr/helper.go @@ -6,7 +6,9 @@ import ( "time" "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-testing-framework/blockchain" + client2 "github.com/smartcontractkit/chainlink-testing-framework/client" "github.com/smartcontractkit/chainlink/integration-tests/actions" "github.com/smartcontractkit/chainlink/integration-tests/client" diff --git a/integration-tests/load/ocr/ocr_test.go b/integration-tests/load/ocr/ocr_test.go index a05f3ad16a2..6bf1487125d 100644 --- a/integration-tests/load/ocr/ocr_test.go +++ b/integration-tests/load/ocr/ocr_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/smartcontractkit/chainlink/integration-tests/k8s" + "github.com/smartcontractkit/wasp" "github.com/stretchr/testify/require" @@ -19,7 +20,7 @@ var ( func TestOCRPerformance(t *testing.T) { l := logging.GetTestLogger(t) - cc, msClient, cd, bootstrapNode, workerNodes, err := k8s.ConnectRemote(l, nil) + cc, msClient, cd, bootstrapNode, workerNodes, err := k8s.ConnectRemote(l) require.NoError(t, err) lt, err := SetupCluster(cc, cd, workerNodes) require.NoError(t, err) @@ -47,7 +48,7 @@ func TestOCRPerformance(t *testing.T) { func TestOCRCapacity(t *testing.T) { l := logging.GetTestLogger(t) - cc, msClient, cd, bootstrapNode, workerNodes, err := k8s.ConnectRemote(l, nil) + cc, msClient, cd, bootstrapNode, workerNodes, err := k8s.ConnectRemote(l) require.NoError(t, err) lt, err := SetupCluster(cc, cd, workerNodes) require.NoError(t, err) diff --git a/integration-tests/load/ocr/vu.go b/integration-tests/load/ocr/vu.go index c8dcdcd3948..a905ec011df 100644 --- a/integration-tests/load/ocr/vu.go +++ b/integration-tests/load/ocr/vu.go @@ -6,13 +6,16 @@ import ( "time" "github.com/rs/zerolog" + "github.com/smartcontractkit/chainlink-testing-framework/blockchain" + + "github.com/smartcontractkit/wasp" + "go.uber.org/ratelimit" + client2 "github.com/smartcontractkit/chainlink-testing-framework/client" "github.com/smartcontractkit/chainlink/integration-tests/actions" "github.com/smartcontractkit/chainlink/integration-tests/client" "github.com/smartcontractkit/chainlink/integration-tests/contracts" - "github.com/smartcontractkit/wasp" - "go.uber.org/ratelimit" ) // VU is a virtual user for the OCR load test