Add CPU Manager related configs

Add two new config files:
- rke2-99-z00-harvester-reserved-resources.yaml:
  which includes system-reserved and kube-reserved CPU resources, both
  of which follow the GKE CPU reservation formula for calculating
  reserved CPU resources
- rke2-99-z01-harvester-cpu-manager.yaml:
  which include cpu-manager-policy, currently the value is "none"

Signed-off-by: Cooper Tseng <[email protected]>

pkg/config/config.go

@@ -3,6 +3,7 @@ package config
 import (
 	"fmt"
 	"net"
+	"runtime"
 	"strings"
 
 	"github.com/imdario/mergo"
@@ -290,6 +291,18 @@ func (c *HarvesterConfig) GetKubeletArgs() ([]string, error) {
 	return args, nil
 }
 
+func (c *HarvesterConfig) GetSystemReserved() (string, error) {
+	return fmt.Sprintf("system-reserved=cpu=%dm", calculateCPUReservedInMilliCPU(runtime.NumCPU(), MaxPods)), nil
+}
+
+func (c *HarvesterConfig) GetKubeReserved() (string, error) {
+	return fmt.Sprintf("kube-reserved=cpu=%dm", calculateCPUReservedInMilliCPU(runtime.NumCPU(), MaxPods)), nil
+}
+
+func (c *HarvesterConfig) GetCPUManagerPolicy() (string, error) {
+	return "cpu-manager-policy=none", nil
+}
+
 func (c HarvesterConfig) ShouldCreateDataPartitionOnOsDisk() bool {
 	// DataDisk is empty means only using the OS disk, and most of the time we should create data
 	// partition on OS disk, unless when ForceMBR=true then we should not create data partition.
@@ -434,3 +447,39 @@ func GenerateRancherdConfig(config *HarvesterConfig) (*yipSchema.YipConfig, erro
 
 	return conf, nil
 }
+
+// inspired by GKE CPU reservations https://cloud.google.com/kubernetes-engine/docs/concepts/plan-node-sizes
+func calculateCPUReservedInMilliCPU(cores int, maxPods int) int64 {
+	// this shouldn't happen
+	if cores <= 0 || maxPods <= 0 {
+		return 0
+	}
+
+	var reserved float64
+
+	// 6% of the first core
+	reserved += float64(6) / 100
+
+	// 1% of the next core (up to 2 cores)
+	if cores > 1 {
+		reserved += float64(1) / 100
+	}
+
+	// 0.5% of the next 2 cores (up to 4 cores)
+	if cores > 2 {
+		reserved += float64(2) * float64(0.5) / 100
+	}
+
+	// 0.25% of any cores above 4 cores
+	if cores > 4 {
+		reserved += float64(cores-4) * float64(0.25) / 100
+	}
+
+	// if the maximum number of Pods per node beyond the default of 110,
+	// reserves an extra 400 mCPU in addition to the preceding reservations.
+	if maxPods > 110 {
+		reserved += 0.4
+	}
+
+	return int64(reserved * 1000)
+}

pkg/config/config_test.go

@@ -1,6 +1,8 @@
 package config
 
 import (
+	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -450,3 +452,90 @@ func TestHarvesterConfigMerge_Addons(t *testing.T) {
 	assert.Equal(t, "the value to overwrite original", conf.Addons["rancher-logging"].ValuesContent, "Addons ValuesContent should be merged")
 	assert.Equal(t, false, conf.Addons["rancher-monitoring"].Enabled, "Addons Enabled false should be merged")
 }
+
+func TestHarvesterReservedResourcesConfigRendering(t *testing.T) {
+	conf := &HarvesterConfig{}
+	content, err := render("rke2-99-z00-harvester-reserved-resources.yaml", conf)
+	assert.Nil(t, err)
+
+	loadedConf := map[string][]string{}
+
+	err = yaml.Unmarshal([]byte(content), &loadedConf)
+	assert.Nil(t, err)
+	assert.Equal(t, 2, len(loadedConf["kubelet-arg+"]))
+	systemReserved := loadedConf["kubelet-arg+"][0]
+	assert.True(t, strings.HasPrefix(systemReserved, "system-reserved=cpu="),
+		fmt.Sprintf("%s doesn't started with system-reserved=cpu=", systemReserved))
+	kubeReserved := loadedConf["kubelet-arg+"][1]
+	assert.True(t, strings.HasPrefix(kubeReserved, "kube-reserved=cpu="),
+		fmt.Sprintf("%s doesn't started with kube-reserved=cpu=", kubeReserved))
+}
+
+func TestHarvesterCPUManagerConfigRendering(t *testing.T) {
+	conf := &HarvesterConfig{}
+	content, err := render("rke2-99-z01-harvester-cpu-manager.yaml", conf)
+	assert.Nil(t, err)
+
+	loadedConf := map[string][]string{}
+
+	err = yaml.Unmarshal([]byte(content), &loadedConf)
+	assert.Nil(t, err)
+	assert.Equal(t, 1, len(loadedConf["kubelet-arg+"]))
+	assert.Equal(t, "cpu-manager-policy=none", loadedConf["kubelet-arg+"][0])
+}
+
+func TestCalculateCPUReservedInMilliCPU(t *testing.T) {
+	testCases := []struct {
+		name               string
+		coreNum            int
+		maxPods            int
+		reservedMilliCores int64
+	}{
+		{
+			name:               "invalid core num",
+			coreNum:            -1,
+			maxPods:            MaxPods,
+			reservedMilliCores: 0,
+		},
+		{
+			name:               "invalid max pods",
+			coreNum:            1,
+			maxPods:            -1,
+			reservedMilliCores: 0,
+		},
+		{
+			name:               "core = 1 and max pods = 110",
+			coreNum:            1,
+			maxPods:            110,
+			reservedMilliCores: 60,
+		},
+		{
+			name:               "core = 1",
+			coreNum:            1,
+			maxPods:            MaxPods,
+			reservedMilliCores: 60 + 400,
+		},
+		{
+			name:               "core = 2",
+			coreNum:            2,
+			maxPods:            MaxPods,
+			reservedMilliCores: 60 + 10 + 400,
+		},
+		{
+			name:               "core = 4",
+			coreNum:            4,
+			maxPods:            MaxPods,
+			reservedMilliCores: 60 + 10 + 5*2 + 400,
+		},
+		{
+			name:               "core = 8",
+			coreNum:            8,
+			maxPods:            MaxPods,
+			reservedMilliCores: 60 + 10 + 5*2 + 2.5*4 + 400,
+		},
+	}
+
+	for _, tc := range testCases {
+		assert.Equal(t, tc.reservedMilliCores, calculateCPUReservedInMilliCPU(tc.coreNum, tc.maxPods))
+	}
+}

pkg/config/cos.go

@@ -397,6 +397,36 @@ func initRancherdStage(config *HarvesterConfig, stage *yipSchema.Stage) error {
 		)
 	}
 
+	systemReservedConfig, err := render("rke2-99-z00-harvester-reserved-resources.yaml", config)
+	if err != nil {
+		return err
+	}
+
+	stage.Files = append(stage.Files,
+		yipSchema.File{
+			Path:        "/etc/rancher/rke2/config.yaml.d/99-z00-harvester-reserved-resources.yaml",
+			Content:     systemReservedConfig,
+			Permissions: 0600,
+			Owner:       0,
+			Group:       0,
+		},
+	)
+
+	cpuManagerPolicyConfig, err := render("rke2-99-z01-harvester-cpu-manager.yaml", config)
+	if err != nil {
+		return err
+	}
+
+	stage.Files = append(stage.Files,
+		yipSchema.File{
+			Path:        "/etc/rancher/rke2/config.yaml.d/99-z01-harvester-cpu-manager.yaml",
+			Content:     cpuManagerPolicyConfig,
+			Permissions: 0600,
+			Owner:       0,
+			Group:       0,
+		},
+	)
+
 	return nil
 }
 

pkg/config/templates/rke2-99-z00-harvester-reserved-resources.yaml

@@ -0,0 +1,3 @@
+kubelet-arg+:
+- {{ printf "%q" .GetSystemReserved }}
+- {{ printf "%q" .GetKubeReserved }}

pkg/config/templates/rke2-99-z01-harvester-cpu-manager.yaml

@@ -0,0 +1,2 @@
+kubelet-arg+:
+- {{ printf "%q" .GetCPUManagerPolicy }}