support proxy env

Signed-off-by: Zhiwei Yin <[email protected]>

go.mod

@@ -15,8 +15,8 @@ require (
 	k8s.io/client-go v0.27.4
 	k8s.io/klog/v2 v2.90.1
 	k8s.io/utils v0.0.0-20230209194617-a36077c30491
-	open-cluster-management.io/addon-framework v0.8.0
-	open-cluster-management.io/api v0.12.0
+	open-cluster-management.io/addon-framework v0.8.1-0.20240102072130-44852ea0722f
+	open-cluster-management.io/api v0.12.1-0.20231124093603-63d09c6ed591
 	sigs.k8s.io/apiserver-network-proxy v0.1.6
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.6
 	sigs.k8s.io/controller-runtime v0.15.0

go.sum

@@ -391,10 +391,10 @@ k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5F
 k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg=
 k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY=
 k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-open-cluster-management.io/addon-framework v0.8.0 h1:i1OReMHuZIoAw2Q04SLjkieU25DnxYilzVZzBNyROwU=
-open-cluster-management.io/addon-framework v0.8.0/go.mod h1:20DP06VXhJ9RE1PetAMEQyeFCP7+nhs92pCAkqbWUOg=
-open-cluster-management.io/api v0.12.0 h1:sNkj4k2XyWA/GLsTiFg82bLIZ7JDZKkLLLyZjJUlJMs=
-open-cluster-management.io/api v0.12.0/go.mod h1:/CZhelEH+30/pX7vXGSZOzLMX0zvjthYOkT/5ZTzVTQ=
+open-cluster-management.io/addon-framework v0.8.1-0.20240102072130-44852ea0722f h1:R5IbFdM4g+kDyONZ1/q67/czRwgDkq9Q7ETZ3RhGaAg=
+open-cluster-management.io/addon-framework v0.8.1-0.20240102072130-44852ea0722f/go.mod h1:aj97pgpGJ0/LpQzBVtU2oDFqqIiZLOPnsjLKG/sVkFw=
+open-cluster-management.io/api v0.12.1-0.20231124093603-63d09c6ed591 h1:l+1R2TxhuXXGA1cDe8qERx+an/e58aIcqi74RAwVUGg=
+open-cluster-management.io/api v0.12.1-0.20231124093603-63d09c6ed591/go.mod h1:/CZhelEH+30/pX7vXGSZOzLMX0zvjthYOkT/5ZTzVTQ=
 sigs.k8s.io/apiserver-network-proxy v0.1.6 h1:mioIpBKJxrBzYZEXZoACPSlh8jLL6cHgxc8u5S/IG5s=
 sigs.k8s.io/apiserver-network-proxy v0.1.6/go.mod h1:0QogIFkEEkRakwcRQBngpOwtR9oKF/LTO9F3er5pfio=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.6 h1:qSYFayz6Wmiy4UI8L/ApIm+4dLlLY0v1OX6AMP2g/2Y=

pkg/common/help.go

@@ -0,0 +1,48 @@
+package common
+
+import (
+	"bytes"
+	"crypto/x509"
+	"encoding/pem"
+	certutil "k8s.io/client-go/util/cert"
+	"reflect"
+)
+
+func MergeCertificateData(caBundles ...[]byte) ([]byte, error) {
+	var all []*x509.Certificate
+	for _, caBundle := range caBundles {
+		if len(caBundle) == 0 {
+			continue
+		}
+
+		certs, err := certutil.ParseCertsPEM(caBundle)
+		if err != nil {
+			return []byte{}, err
+		}
+		all = append(all, certs...)
+	}
+
+	// remove duplicated cert
+	var merged []*x509.Certificate
+	for i := range all {
+		found := false
+		for j := range merged {
+			if reflect.DeepEqual(all[i].Raw, merged[j].Raw) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			merged = append(merged, all[i])
+		}
+	}
+
+	// encode the merged certificates
+	b := bytes.Buffer{}
+	for _, cert := range merged {
+		if err := pem.Encode(&b, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}); err != nil {
+			return []byte{}, err
+		}
+	}
+	return b.Bytes(), nil
+}

pkg/proxyagent/agent/agent.go

@@ -134,9 +134,9 @@ func NewAgentAddon(
 		).
 		WithGetValuesFuncs(
 			GetClusterProxyValueFunc(runtimeClient, nativeClient, signerNamespace, caCertData, v1CSRSupported, enableKubeApiProxy),
-			addonfactory.GetAddOnDeloymentConfigValues(
-				addonfactory.NewAddOnDeloymentConfigGetter(addonClient),
-				toAgentAddOnChartValues,
+			addonfactory.GetAddOnDeploymentConfigValues(
+				utils.NewAddOnDeploymentConfigGetter(addonClient),
+				toAgentAddOnChartValues(caCertData),
 			),
 		)
 
@@ -166,7 +166,7 @@ func GetClusterProxyValueFunc(
 		}
 
 		// only handle there is only one managed proxy configuration for one addon
-		// TODO may consider to handle mutiple managed proxy configurations for one addon
+		// TODO may consider to handle multiple managed proxy configurations for one addon
 		if len(managedProxyConfigurations) != 1 {
 			return nil, fmt.Errorf("unexpected managed proxy configurations: %v", managedProxyConfigurations)
 		}
@@ -300,7 +300,7 @@ func GetClusterProxyValueFunc(
 			"includeStaticProxyAgentSecret": !v1CSRSupported,
 			"staticProxyAgentSecretCert":    certDataBase64,
 			"staticProxyAgentSecretKey":     keyDataBase64,
-			// support to access not only but also other other services on managed cluster
+			// support to access not only but also other services on managed cluster
 			"agentIdentifiers":   agentIdentifiers,
 			"servicesToExpose":   servicesToExpose,
 			"enableKubeApiProxy": enableKubeApiProxy,
@@ -405,16 +405,38 @@ func removeDupAndSortServices(services []serviceToExpose) []serviceToExpose {
 	return newServices
 }
 
-func toAgentAddOnChartValues(config addonv1alpha1.AddOnDeploymentConfig) (addonfactory.Values, error) {
-	values := addonfactory.Values{}
-	for _, variable := range config.Spec.CustomizedVariables {
-		values[variable.Name] = variable.Value
-	}
+func toAgentAddOnChartValues(caCertData []byte) func(config addonv1alpha1.AddOnDeploymentConfig) (addonfactory.Values, error) {
+	return func(config addonv1alpha1.AddOnDeploymentConfig) (addonfactory.Values, error) {
+		values := addonfactory.Values{}
+		for _, variable := range config.Spec.CustomizedVariables {
+			values[variable.Name] = variable.Value
+		}
 
-	if config.Spec.NodePlacement != nil {
-		values["nodeSelector"] = config.Spec.NodePlacement.NodeSelector
-		values["tolerations"] = config.Spec.NodePlacement.Tolerations
-	}
+		if config.Spec.NodePlacement != nil {
+			values["nodeSelector"] = config.Spec.NodePlacement.NodeSelector
+			values["tolerations"] = config.Spec.NodePlacement.Tolerations
+		}
+
+		proxyConfig := config.Spec.ProxyConfig
+		values["proxyConfig"] = map[string]string{
+			"HTTP_PROXY":  proxyConfig.HTTPProxy,
+			"HTTPS_PROXY": proxyConfig.HTTPSProxy,
+			"NO_PROXY":    proxyConfig.NoProxy,
+		}
+
+		if proxyConfig.HTTPSProxy != "" && len(proxyConfig.CABundle) != 0 {
+			rawProxyCaCert, err := base64.StdEncoding.DecodeString(string(proxyConfig.CABundle))
+			if err != nil {
+				return nil, fmt.Errorf("faield to decdoe proxy env ca. %v", err)
+			}
 
-	return values, nil
+			caCert, err := common.MergeCertificateData(rawProxyCaCert, caCertData)
+			if err != nil {
+				return nil, fmt.Errorf("faield to merge proxy env ca. %v", err)
+			}
+
+			values["base64EncodedCAData"] = base64.StdEncoding.EncodeToString(caCert)
+		}
+		return values, nil
+	}
 }

pkg/proxyagent/agent/agent_test.go

@@ -51,6 +51,7 @@ func init() {
 	testscheme.AddKnownTypes(proxyv1alpha1.SchemeGroupVersion, &proxyv1alpha1.ManagedProxyConfiguration{})
 	testscheme.AddKnownTypes(clusterv1beta2.SchemeGroupVersion, &clusterv1beta2.ManagedClusterSetList{})
 	testscheme.AddKnownTypes(proxyv1alpha1.SchemeGroupVersion, &proxyv1alpha1.ManagedProxyServiceResolverList{})
+	testscheme.AddKnownTypes(addonv1alpha1.SchemeGroupVersion, &addonv1alpha1.AddOnDeploymentConfig{})
 }
 
 func TestFilterMPSR(t *testing.T) {
@@ -614,6 +615,19 @@ func TestNewAgentAddon(t *testing.T) {
 				assert.NotNil(t, agentDeploy)
 				assert.Equal(t, nodeSelector, agentDeploy.Spec.Template.Spec.NodeSelector)
 				assert.Equal(t, tolerations, agentDeploy.Spec.Template.Spec.Tolerations)
+				envCount := 0
+				for _, container := range agentDeploy.Spec.Template.Spec.Containers {
+					if container.Name == "proxy-agent" {
+						envCount = len(container.Env)
+					}
+				}
+				assert.Equal(t, 0, envCount)
+				caSecret := getCASecret(manifests)
+				assert.NotNil(t, caSecret)
+				caCrt := string(caSecret.Data["ca.crt"])
+				count := strings.Count(caCrt, "-----BEGIN CERTIFICATE-----")
+				assert.Equal(t, 1, count)
+
 			},
 		},
 		{
@@ -660,6 +674,40 @@ func TestNewAgentAddon(t *testing.T) {
 				assert.ElementsMatch(t, expectedManifestNamesWithoutClusterService, manifestNames(manifests))
 			},
 		},
+		{
+			name:    "with addon deployment config including proxy config",
+			cluster: newCluster(clusterName, true),
+			addon: func() *addonv1alpha1.ManagedClusterAddOn {
+				addOn := newAddOn(addOnName, clusterName)
+				addOn.Status.ConfigReferences = []addonv1alpha1.ConfigReference{
+					newManagedProxyConfigReference(managedProxyConfigName),
+					newAddOndDeploymentConfigReference(addOndDeployConfigName, clusterName),
+				}
+				return addOn
+			}(),
+			managedProxyConfigs:     []runtimeclient.Object{newManagedProxyConfig(managedProxyConfigName, proxyv1alpha1.EntryPointTypePortForward)},
+			addOndDeploymentConfigs: []runtime.Object{newAddOnDeploymentConfigWithProxy(addOndDeployConfigName, clusterName)},
+			v1CSRSupported:          true,
+			enableKubeApiProxy:      true,
+			verifyManifests: func(t *testing.T, manifests []runtime.Object) {
+				assert.Len(t, manifests, len(expectedManifestNames))
+				assert.ElementsMatch(t, expectedManifestNames, manifestNames(manifests))
+				agentDeploy := getAgentDeployment(manifests)
+				assert.NotNil(t, agentDeploy)
+				envCount := 0
+				for _, container := range agentDeploy.Spec.Template.Spec.Containers {
+					if container.Name == "proxy-agent" {
+						envCount = len(container.Env)
+					}
+				}
+				assert.Equal(t, 4, envCount)
+				caSecret := getCASecret(manifests)
+				assert.NotNil(t, caSecret)
+				caCrt := string(caSecret.Data["ca.crt"])
+				count := strings.Count(caCrt, "-----BEGIN CERTIFICATE-----")
+				assert.Equal(t, 2, count)
+			},
+		},
 	}
 
 	for _, c := range cases {
@@ -880,6 +928,29 @@ func newAddOnDeploymentConfigWithCustomizedServiceDomain(name, namespace, servic
 	}
 }
 
+var fakeCA = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM2VENDQWRFQ0ZHSG5lTUpBQ1NjR2lRSnA2K1RYa0NKRVBTVitNQTBHQ1NxR1NJYjNEUUVCQ3dVQU1ERXgKRmpBVUJnTlZCQW9NRFU5d1pXNVRhR2xtZENCQlEwMHhGekFWQmdOVkJBTU1EbmQzZHk1eVpXUm9ZWFF1WTI5dApNQjRYRFRJek1URXhNakV5TURZME4xb1hEVEkwTVRFeE1URXlNRFkwTjFvd01URVdNQlFHQTFVRUNnd05UM0JsCmJsTm9hV1owSUVGRFRURVhNQlVHQTFVRUF3d09kM2QzTG5KbFpHaGhkQzVqYjIwd2dnRWlNQTBHQ1NxR1NJYjMKRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFEUXZMbHFjYXpYZmxXNXgzcVFDSE52ZjNqTFNCY0QrY3pCczFoMApUV0p2TWEvWVd2T2MrK3VNWXg2OW1RaXRCWEFaMEsyUVpQa1BYK2lEc244Mk9mNklYTUpUSVpmZk1Wb3g4UmtqCkNlQ00vdlNaMzExVGlwa0NkaGVTbnp0WElhek1hN0ZZS3BVT2htYTF3L2RReFcvcnIwandwRG9TMFUvN0xhWGwKNHF2bUF4Wk1iSHVWaFk2S0RZSGJ2MEdKYWdqekJtVkpieTZlMFg3MkozL05ZME1KT2plYklrOTEydjBXZ1pUKwo3UWU0a29scVY1MkQvaUhYV0xFUzhXMWQrMFZUbnlRaFAzY3RvNWp3TFZyWnQ2NDFZL0lRc2ZNQ0w1bGdhVTF0Cm9UMlcvQ3F1amw5aCt0UCt2SG1rNk5JZXk2RUNIdm1MV0xLbU5nblp2M0d0bVdnZEFnTUJBQUV3RFFZSktvWkkKaHZjTkFRRUxCUUFEZ2dFQkFKSjBnd0UxSUR4SlNzaUd1TGxDMlVGV2J3U0RHMUVEK3VlQWYvRDRlV0VSWFZDUAo4aVdZZC9RckdsakYxNGxvZllHb280Vk5PL28xQWJQS2gveXB4UW16REdrVE1NaGg2WFg1bExob3RZWHZERlM2CmlkQXk5TFpiWDFUQnV5UEcwNmorbkI4eEtEY3F4aFNLYTlNb0trck9XcmtGbnFZS2syQzIyZGRvZVlZdlRjR2cKK2JmZ3RSWFJRUFdQRmt2NDR5MGlMZVh0S0VMbHBQMkMyQW5JQkU4b2hzY0JiYnloVmptem5YS1dFSTg3T0xmUgoxNDJBOWoydlVVQW80T0o5d1JCei8raDFXUXkyL3prclVUMW90MFdienY1cy91YmlUQkRpSjlQQ0k4YkZmZXplCnpDbCthbEE5aUFJdGt4OVdZS2pzaDFuVHEzTnJwVWM0MXBJWlFBQT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo="
+
+func newAddOnDeploymentConfigWithProxy(name, namespace string) *addonv1alpha1.AddOnDeploymentConfig {
+	return &addonv1alpha1.AddOnDeploymentConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: addonv1alpha1.AddOnDeploymentConfigSpec{
+			NodePlacement: &addonv1alpha1.NodePlacement{
+				Tolerations:  tolerations,
+				NodeSelector: nodeSelector,
+			},
+			ProxyConfig: addonv1alpha1.ProxyConfig{
+				HTTPProxy:  "http://192.168.1.1",
+				HTTPSProxy: "https://192.168.1.1",
+				CABundle:   []byte(fakeCA),
+				NoProxy:    "localhost",
+			},
+		},
+	}
+}
+
 func newLoadBalancerService(ingress string) *corev1.Service {
 	svc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
@@ -950,3 +1021,16 @@ func getProxyServerHost(deploy *appsv1.Deployment) string {
 	}
 	return ""
 }
+
+func getCASecret(manifests []runtime.Object) *corev1.Secret {
+	for _, manifest := range manifests {
+		switch obj := manifest.(type) {
+		case *corev1.Secret:
+			if obj.Name == "cluster-proxy-ca" {
+				return obj
+			}
+		}
+	}
+
+	return nil
+}

pkg/proxyagent/agent/manifests/charts/addon-agent/templates/addon-agent-deployment.yaml

@@ -59,6 +59,21 @@ spec:
             privileged: false
             runAsNonRoot: true
             readOnlyRootFilesystem: true
+          env:
+            {{- if .Values.proxyConfig.HTTP_PROXY }}
+            - name: HTTP_PROXY
+              value: {{ .Values.proxyConfig.HTTP_PROXY }}
+            {{- end }}
+            {{- if .Values.proxyConfig.HTTPS_PROXY }}
+            - name: HTTPS_PROXY
+              value: {{ .Values.proxyConfig.HTTPS_PROXY }}
+            - name: ROOT_CA_CERT
+              value: "/etc/ca/ca.crt"
+            {{- end }}
+            {{- if .Values.proxyConfig.NO_PROXY }}
+            - name: NO_PROXY
+              value: {{ .Values.proxyConfig.NO_PROXY }}
+            {{- end }}
           volumeMounts:
             - name: ca
               mountPath: /etc/ca

pkg/proxyagent/agent/manifests/charts/addon-agent/values.yaml

@@ -33,3 +33,7 @@ serviceDomain: ""
 tolerations: []
 
 nodeSelector: {}
+proxyConfig:
+  HTTP_PROXY: null
+  HTTPS_PROXY: null
+  NO_PROXY: null