fix(core, kubevirt): add ability to configure burst for virt-api rate…

… limiter * fix(core, kubevirt): add ability to configure burst for virt-api rate limiter * chore(core, kubevirt): add default values of QPS and burst to logs When migrating a big number of virtual machines (>50), we encounter a bottleneck in virt-api in the form of a rate limiter that prevents migration completion. A patch has been updated to virt-api, enabling the configuration of Burst for the rate limiter via an environment variable VIRT_API_RATE_LIMITER_BURST. Signed-off-by: Isteb4k <[email protected]> --------- Signed-off-by: Isteb4k <[email protected]>
deckhouse · May 27, 2024 · e5c4605 · e5c4605
1 parent 7c7fb60
commit e5c4605
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 7 deletions.
diff --git a/...tches/013-virt-api-rate-limiter-qps.patch → ...t/patches/013-virt-api-rate-limiter.patch b/...tches/013-virt-api-rate-limiter-qps.patch → ...t/patches/013-virt-api-rate-limiter.patch
@@ -1,5 +1,5 @@
 diff --git a/pkg/virt-api/api.go b/pkg/virt-api/api.go
-index 120f2d68f..3c11d4fc3 100644
+index 120f2d68f..5a92cbaa4 100644
 --- a/pkg/virt-api/api.go
 +++ b/pkg/virt-api/api.go
 @@ -27,6 +27,7 @@ import (
@@ -10,16 +10,17 @@ index 120f2d68f..3c11d4fc3 100644
  	"sync"
  	"syscall"
  	"time"
-@@ -92,6 +93,8 @@ const (
+@@ -92,6 +93,9 @@ const (
  	httpStatusNotFoundMessage     = "Not Found"
  	httpStatusBadRequestMessage   = "Bad Request"
  	httpStatusInternalServerError = "Internal Server Error"
 +
-+	VirtAPIRateLimiterQPSEnvVar = "VIRT_API_RATE_LIMITER_QPS"
++	VirtAPIRateLimiterQPSEnvVar   = "VIRT_API_RATE_LIMITER_QPS"
++	VirtAPIRateLimiterBurstEnvVar = "VIRT_API_RATE_LIMITER_BURST"
  )
 
  type VirtApi interface {
-@@ -1089,7 +1092,18 @@ func (app *virtAPIApp) shouldChangeLogVerbosity() {
+@@ -1089,8 +1093,29 @@ func (app *virtAPIApp) shouldChangeLogVerbosity() {
  // Update virt-handler rate limiter
  func (app *virtAPIApp) shouldChangeRateLimiter() {
  	config := app.clusterConfig.GetConfig()
@@ -28,13 +29,24 @@ index 120f2d68f..3c11d4fc3 100644
 +	if os.Getenv(VirtAPIRateLimiterQPSEnvVar) != "" {
 +		qpsFromEnv, err := strconv.ParseFloat(os.Getenv(VirtAPIRateLimiterQPSEnvVar), 32)
 +		if err != nil {
-+			log.Log.Errorf("failed to parse %s: %s", VirtAPIRateLimiterQPSEnvVar, err)
++			log.Log.Errorf("failed to parse %s: %s, will use default QPS burst %v", VirtAPIRateLimiterQPSEnvVar, err, qps)
 +		} else {
 +			qps = float32(qpsFromEnv)
 +			log.Log.V(2).Infof("use rate limiter QPS %v from %s", qps, VirtAPIRateLimiterQPSEnvVar)
 +		}
 +	}
 +
  	burst := config.APIConfiguration.RestClient.RateLimiter.TokenBucketRateLimiter.Burst
++	if os.Getenv(VirtAPIRateLimiterBurstEnvVar) != "" {
++		burstFromEnv, err := strconv.ParseInt(os.Getenv(VirtAPIRateLimiterBurstEnvVar), 10, 32)
++		if err != nil {
++			log.Log.Errorf("failed to parse %s: %s, will use default burst %d", VirtAPIRateLimiterBurstEnvVar, err, burst)
++		} else {
++			burst = int(burstFromEnv)
++			log.Log.V(2).Infof("use rate limiter burst %v from %s", burst, VirtAPIRateLimiterBurstEnvVar)
++		}
++	}
++
  	app.reloadableRateLimiter.Set(flowcontrol.NewTokenBucketRateLimiter(qps, burst))
  	log.Log.V(2).Infof("setting rate limiter for the API to %v QPS and %v Burst", qps, burst)
+ 	qps = config.WebhookConfiguration.RestClient.RateLimiter.TokenBucketRateLimiter.QPS
diff --git a/images/virt-artifact/patches/README.md b/images/virt-artifact/patches/README.md
@@ -32,8 +32,8 @@ Added the ability for virt-api to authenticate clients with certificates signed
 #### `012-support-kubeconfig-env.patch`
 Support `KUBECONFIG` environment variable. 
 
-#### `013-virt-api-rate-limiter-qps.patch`
-A patch has been added to enable the configuration of QPS for the rate limiter via an environment variable VIRT_API_RATE_LIMITER_QPS.
+#### `013-virt-api-rate-limiter.patch`
+A patch has been added to enable the configuration of the rate limiter via the environment variables VIRT_API_RATE_LIMITER_QPS and VIRT_API_RATE_LIMITER_BURST.
 
 #### `014-delete-apiserver.patch`
 Do not create Kubevirt APIService.
diff --git a/templates/kubevirt/kubevirt.yaml b/templates/kubevirt/kubevirt.yaml
@@ -266,6 +266,9 @@ spec:
                   "env": [{
                     "name": "VIRT_API_RATE_LIMITER_QPS",
                     "value": "5000"
+                  },{
+                    "name": "VIRT_API_RATE_LIMITER_BURST",
+                    "value": "6000"
                   }]
                 }]
               }