diff --git a/pkg/config/config.go b/pkg/config/config.go index 54947db77..743daf5ef 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -3,6 +3,7 @@ package config import ( "fmt" "net" + "runtime" "strings" "github.com/imdario/mergo" @@ -290,6 +291,18 @@ func (c *HarvesterConfig) GetKubeletArgs() ([]string, error) { return args, nil } +func (c *HarvesterConfig) GetSystemReserved() (string, error) { + return fmt.Sprintf("system-reserved=cpu=%dm", calculateCPUReservedInMilliCPU(runtime.NumCPU(), MaxPods)), nil +} + +func (c *HarvesterConfig) GetKubeReserved() (string, error) { + return fmt.Sprintf("kube-reserved=cpu=%dm", calculateCPUReservedInMilliCPU(runtime.NumCPU(), MaxPods)), nil +} + +func (c *HarvesterConfig) GetCPUManagerPolicy() (string, error) { + return "cpu-manager-policy=none", nil +} + func (c HarvesterConfig) ShouldCreateDataPartitionOnOsDisk() bool { // DataDisk is empty means only using the OS disk, and most of the time we should create data // partition on OS disk, unless when ForceMBR=true then we should not create data partition. @@ -434,3 +447,39 @@ func GenerateRancherdConfig(config *HarvesterConfig) (*yipSchema.YipConfig, erro return conf, nil } + +// inspired by GKE CPU reservations https://cloud.google.com/kubernetes-engine/docs/concepts/plan-node-sizes +func calculateCPUReservedInMilliCPU(cores int, maxPods int) int64 { + // this shouldn't happen + if cores <= 0 || maxPods <= 0 { + return 0 + } + + var reserved float64 + + // 6% of the first core + reserved += float64(6) / 100 + + // 1% of the next core (up to 2 cores) + if cores > 1 { + reserved += float64(1) / 100 + } + + // 0.5% of the next 2 cores (up to 4 cores) + if cores > 2 { + reserved += float64(2) * float64(0.5) / 100 + } + + // 0.25% of any cores above 4 cores + if cores > 4 { + reserved += float64(cores-4) * float64(0.25) / 100 + } + + // if the maximum number of Pods per node beyond the default of 110, + // reserves an extra 400 mCPU in addition to the preceding reservations. + if maxPods > 110 { + reserved += 0.4 + } + + return int64(reserved * 1000) +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 9cf4677d2..bc3df91ae 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -1,6 +1,8 @@ package config import ( + "fmt" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -450,3 +452,90 @@ func TestHarvesterConfigMerge_Addons(t *testing.T) { assert.Equal(t, "the value to overwrite original", conf.Addons["rancher-logging"].ValuesContent, "Addons ValuesContent should be merged") assert.Equal(t, false, conf.Addons["rancher-monitoring"].Enabled, "Addons Enabled false should be merged") } + +func TestHarvesterReservedResourcesConfigRendering(t *testing.T) { + conf := &HarvesterConfig{} + content, err := render("rke2-99-z00-harvester-reserved-resources.yaml", conf) + assert.Nil(t, err) + + loadedConf := map[string][]string{} + + err = yaml.Unmarshal([]byte(content), &loadedConf) + assert.Nil(t, err) + assert.Equal(t, 2, len(loadedConf["kubelet-arg+"])) + systemReserved := loadedConf["kubelet-arg+"][0] + assert.True(t, strings.HasPrefix(systemReserved, "system-reserved=cpu="), + fmt.Sprintf("%s doesn't started with system-reserved=cpu=", systemReserved)) + kubeReserved := loadedConf["kubelet-arg+"][1] + assert.True(t, strings.HasPrefix(kubeReserved, "kube-reserved=cpu="), + fmt.Sprintf("%s doesn't started with kube-reserved=cpu=", kubeReserved)) +} + +func TestHarvesterCPUManagerConfigRendering(t *testing.T) { + conf := &HarvesterConfig{} + content, err := render("rke2-99-z01-harvester-cpu-manager.yaml", conf) + assert.Nil(t, err) + + loadedConf := map[string][]string{} + + err = yaml.Unmarshal([]byte(content), &loadedConf) + assert.Nil(t, err) + assert.Equal(t, 1, len(loadedConf["kubelet-arg+"])) + assert.Equal(t, "cpu-manager-policy=none", loadedConf["kubelet-arg+"][0]) +} + +func TestCalculateCPUReservedInMilliCPU(t *testing.T) { + testCases := []struct { + name string + coreNum int + maxPods int + reservedMilliCores int64 + }{ + { + name: "invalid core num", + coreNum: -1, + maxPods: MaxPods, + reservedMilliCores: 0, + }, + { + name: "invalid max pods", + coreNum: 1, + maxPods: -1, + reservedMilliCores: 0, + }, + { + name: "core = 1 and max pods = 110", + coreNum: 1, + maxPods: 110, + reservedMilliCores: 60, + }, + { + name: "core = 1", + coreNum: 1, + maxPods: MaxPods, + reservedMilliCores: 60 + 400, + }, + { + name: "core = 2", + coreNum: 2, + maxPods: MaxPods, + reservedMilliCores: 60 + 10 + 400, + }, + { + name: "core = 4", + coreNum: 4, + maxPods: MaxPods, + reservedMilliCores: 60 + 10 + 5*2 + 400, + }, + { + name: "core = 8", + coreNum: 8, + maxPods: MaxPods, + reservedMilliCores: 60 + 10 + 5*2 + 2.5*4 + 400, + }, + } + + for _, tc := range testCases { + assert.Equal(t, tc.reservedMilliCores, calculateCPUReservedInMilliCPU(tc.coreNum, tc.maxPods)) + } +} diff --git a/pkg/config/cos.go b/pkg/config/cos.go index 3e550a1ab..fd06eea20 100644 --- a/pkg/config/cos.go +++ b/pkg/config/cos.go @@ -397,6 +397,36 @@ func initRancherdStage(config *HarvesterConfig, stage *yipSchema.Stage) error { ) } + systemReservedConfig, err := render("rke2-99-z00-harvester-reserved-resources.yaml", config) + if err != nil { + return err + } + + stage.Files = append(stage.Files, + yipSchema.File{ + Path: "/etc/rancher/rke2/config.yaml.d/99-z00-harvester-reserved-resources.yaml", + Content: systemReservedConfig, + Permissions: 0600, + Owner: 0, + Group: 0, + }, + ) + + cpuManagerPolicyConfig, err := render("rke2-99-z01-harvester-cpu-manager.yaml", config) + if err != nil { + return err + } + + stage.Files = append(stage.Files, + yipSchema.File{ + Path: "/etc/rancher/rke2/config.yaml.d/99-z01-harvester-cpu-manager.yaml", + Content: cpuManagerPolicyConfig, + Permissions: 0600, + Owner: 0, + Group: 0, + }, + ) + return nil } diff --git a/pkg/config/templates/rke2-99-z00-harvester-reserved-resources.yaml b/pkg/config/templates/rke2-99-z00-harvester-reserved-resources.yaml new file mode 100644 index 000000000..6c9ca78f9 --- /dev/null +++ b/pkg/config/templates/rke2-99-z00-harvester-reserved-resources.yaml @@ -0,0 +1,3 @@ +kubelet-arg+: +- {{ printf "%q" .GetSystemReserved }} +- {{ printf "%q" .GetKubeReserved }} \ No newline at end of file diff --git a/pkg/config/templates/rke2-99-z01-harvester-cpu-manager.yaml b/pkg/config/templates/rke2-99-z01-harvester-cpu-manager.yaml new file mode 100644 index 000000000..d26e6ff2e --- /dev/null +++ b/pkg/config/templates/rke2-99-z01-harvester-cpu-manager.yaml @@ -0,0 +1,2 @@ +kubelet-arg+: +- {{ printf "%q" .GetCPUManagerPolicy }} \ No newline at end of file