diff --git a/go.mod b/go.mod index f2aca6bb..70e4c3ca 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( k8s.io/klog/v2 v2.90.1 k8s.io/utils v0.0.0-20230209194617-a36077c30491 open-cluster-management.io/addon-framework v0.8.0 - open-cluster-management.io/api v0.12.0 + open-cluster-management.io/api v0.12.1-0.20231124093603-63d09c6ed591 sigs.k8s.io/apiserver-network-proxy v0.1.6 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.6 sigs.k8s.io/controller-runtime v0.15.0 diff --git a/go.sum b/go.sum index 43a47298..fb1e3681 100644 --- a/go.sum +++ b/go.sum @@ -393,8 +393,8 @@ k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPB k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= open-cluster-management.io/addon-framework v0.8.0 h1:i1OReMHuZIoAw2Q04SLjkieU25DnxYilzVZzBNyROwU= open-cluster-management.io/addon-framework v0.8.0/go.mod h1:20DP06VXhJ9RE1PetAMEQyeFCP7+nhs92pCAkqbWUOg= -open-cluster-management.io/api v0.12.0 h1:sNkj4k2XyWA/GLsTiFg82bLIZ7JDZKkLLLyZjJUlJMs= -open-cluster-management.io/api v0.12.0/go.mod h1:/CZhelEH+30/pX7vXGSZOzLMX0zvjthYOkT/5ZTzVTQ= +open-cluster-management.io/api v0.12.1-0.20231124093603-63d09c6ed591 h1:l+1R2TxhuXXGA1cDe8qERx+an/e58aIcqi74RAwVUGg= +open-cluster-management.io/api v0.12.1-0.20231124093603-63d09c6ed591/go.mod h1:/CZhelEH+30/pX7vXGSZOzLMX0zvjthYOkT/5ZTzVTQ= sigs.k8s.io/apiserver-network-proxy v0.1.6 h1:mioIpBKJxrBzYZEXZoACPSlh8jLL6cHgxc8u5S/IG5s= sigs.k8s.io/apiserver-network-proxy v0.1.6/go.mod h1:0QogIFkEEkRakwcRQBngpOwtR9oKF/LTO9F3er5pfio= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.6 h1:qSYFayz6Wmiy4UI8L/ApIm+4dLlLY0v1OX6AMP2g/2Y= diff --git a/pkg/common/help.go b/pkg/common/help.go new file mode 100644 index 00000000..02af4778 --- /dev/null +++ b/pkg/common/help.go @@ -0,0 +1,48 @@ +package common + +import ( + "bytes" + "crypto/x509" + "encoding/pem" + certutil "k8s.io/client-go/util/cert" + "reflect" +) + +func MergeCertificateData(caBundles ...[]byte) ([]byte, error) { + var all []*x509.Certificate + for _, caBundle := range caBundles { + if len(caBundle) == 0 { + continue + } + + certs, err := certutil.ParseCertsPEM(caBundle) + if err != nil { + return []byte{}, err + } + all = append(all, certs...) + } + + // remove duplicated cert + var merged []*x509.Certificate + for i := range all { + found := false + for j := range merged { + if reflect.DeepEqual(all[i].Raw, merged[j].Raw) { + found = true + break + } + } + if !found { + merged = append(merged, all[i]) + } + } + + // encode the merged certificates + b := bytes.Buffer{} + for _, cert := range merged { + if err := pem.Encode(&b, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}); err != nil { + return []byte{}, err + } + } + return b.Bytes(), nil +} diff --git a/pkg/proxyagent/agent/agent.go b/pkg/proxyagent/agent/agent.go index a37f7769..7cd05171 100644 --- a/pkg/proxyagent/agent/agent.go +++ b/pkg/proxyagent/agent/agent.go @@ -134,9 +134,9 @@ func NewAgentAddon( ). WithGetValuesFuncs( GetClusterProxyValueFunc(runtimeClient, nativeClient, signerNamespace, caCertData, v1CSRSupported, enableKubeApiProxy), - addonfactory.GetAddOnDeloymentConfigValues( - addonfactory.NewAddOnDeloymentConfigGetter(addonClient), - toAgentAddOnChartValues, + addonfactory.GetAddOnDeploymentConfigValues( + utils.NewAddOnDeploymentConfigGetter(addonClient), + toAgentAddOnChartValues(caCertData), ), ) @@ -166,7 +166,7 @@ func GetClusterProxyValueFunc( } // only handle there is only one managed proxy configuration for one addon - // TODO may consider to handle mutiple managed proxy configurations for one addon + // TODO may consider to handle multiple managed proxy configurations for one addon if len(managedProxyConfigurations) != 1 { return nil, fmt.Errorf("unexpected managed proxy configurations: %v", managedProxyConfigurations) } @@ -300,7 +300,7 @@ func GetClusterProxyValueFunc( "includeStaticProxyAgentSecret": !v1CSRSupported, "staticProxyAgentSecretCert": certDataBase64, "staticProxyAgentSecretKey": keyDataBase64, - // support to access not only but also other other services on managed cluster + // support to access not only but also other services on managed cluster "agentIdentifiers": agentIdentifiers, "servicesToExpose": servicesToExpose, "enableKubeApiProxy": enableKubeApiProxy, @@ -405,16 +405,37 @@ func removeDupAndSortServices(services []serviceToExpose) []serviceToExpose { return newServices } -func toAgentAddOnChartValues(config addonv1alpha1.AddOnDeploymentConfig) (addonfactory.Values, error) { - values := addonfactory.Values{} - for _, variable := range config.Spec.CustomizedVariables { - values[variable.Name] = variable.Value - } +func toAgentAddOnChartValues(caCertData []byte) func(config addonv1alpha1.AddOnDeploymentConfig) (addonfactory.Values, error) { + return func(config addonv1alpha1.AddOnDeploymentConfig) (addonfactory.Values, error) { + values := addonfactory.Values{} + for _, variable := range config.Spec.CustomizedVariables { + values[variable.Name] = variable.Value + } - if config.Spec.NodePlacement != nil { - values["nodeSelector"] = config.Spec.NodePlacement.NodeSelector - values["tolerations"] = config.Spec.NodePlacement.Tolerations - } + if config.Spec.NodePlacement != nil { + values["nodeSelector"] = config.Spec.NodePlacement.NodeSelector + values["tolerations"] = config.Spec.NodePlacement.Tolerations + } - return values, nil + proxyConfig := config.Spec.ProxyConfig + values["proxyConfig"] = map[string]string{ + "HTTP_PROXY": proxyConfig.HTTPProxy, + "HTTPS_PROXY": proxyConfig.HTTPSProxy, + "NO_PROXY": proxyConfig.NoProxy, + } + + if proxyConfig.HTTPSProxy != "" && len(proxyConfig.CABundle) != 0 { + rawProxyCaCert, err := base64.StdEncoding.DecodeString(string(proxyConfig.CABundle)) + if err != nil { + return nil, fmt.Errorf("faield to decdoe proxy env ca. %v", err) + } + + caCert, err := common.MergeCertificateData(rawProxyCaCert, caCertData) + if err != nil { + return nil, fmt.Errorf("faield to merge proxy env ca. %v", err) + } + values["base64EncodedCAData"] = base64.StdEncoding.EncodeToString(caCert) + } + return values, nil + } } diff --git a/pkg/proxyagent/agent/agent_test.go b/pkg/proxyagent/agent/agent_test.go index d8a95922..0fee2083 100644 --- a/pkg/proxyagent/agent/agent_test.go +++ b/pkg/proxyagent/agent/agent_test.go @@ -51,6 +51,7 @@ func init() { testscheme.AddKnownTypes(proxyv1alpha1.SchemeGroupVersion, &proxyv1alpha1.ManagedProxyConfiguration{}) testscheme.AddKnownTypes(clusterv1beta2.SchemeGroupVersion, &clusterv1beta2.ManagedClusterSetList{}) testscheme.AddKnownTypes(proxyv1alpha1.SchemeGroupVersion, &proxyv1alpha1.ManagedProxyServiceResolverList{}) + testscheme.AddKnownTypes(addonv1alpha1.SchemeGroupVersion, &addonv1alpha1.AddOnDeploymentConfig{}) } func TestFilterMPSR(t *testing.T) { @@ -614,6 +615,19 @@ func TestNewAgentAddon(t *testing.T) { assert.NotNil(t, agentDeploy) assert.Equal(t, nodeSelector, agentDeploy.Spec.Template.Spec.NodeSelector) assert.Equal(t, tolerations, agentDeploy.Spec.Template.Spec.Tolerations) + envCount := 0 + for _, container := range agentDeploy.Spec.Template.Spec.Containers { + if container.Name == "proxy-agent" { + envCount = len(container.Env) + } + } + assert.Equal(t, 0, envCount) + caSecret := getCASecret(manifests) + assert.NotNil(t, caSecret) + caCrt := string(caSecret.Data["ca.crt"]) + count := strings.Count(caCrt, "-----BEGIN CERTIFICATE-----") + assert.Equal(t, 1, count) + }, }, { @@ -660,6 +674,40 @@ func TestNewAgentAddon(t *testing.T) { assert.ElementsMatch(t, expectedManifestNamesWithoutClusterService, manifestNames(manifests)) }, }, + { + name: "with addon deployment config including proxy config", + cluster: newCluster(clusterName, true), + addon: func() *addonv1alpha1.ManagedClusterAddOn { + addOn := newAddOn(addOnName, clusterName) + addOn.Status.ConfigReferences = []addonv1alpha1.ConfigReference{ + newManagedProxyConfigReference(managedProxyConfigName), + newAddOndDeploymentConfigReference(addOndDeployConfigName, clusterName), + } + return addOn + }(), + managedProxyConfigs: []runtimeclient.Object{newManagedProxyConfig(managedProxyConfigName, proxyv1alpha1.EntryPointTypePortForward)}, + addOndDeploymentConfigs: []runtime.Object{newAddOnDeploymentConfigWithProxy(addOndDeployConfigName, clusterName)}, + v1CSRSupported: true, + enableKubeApiProxy: true, + verifyManifests: func(t *testing.T, manifests []runtime.Object) { + assert.Len(t, manifests, len(expectedManifestNames)) + assert.ElementsMatch(t, expectedManifestNames, manifestNames(manifests)) + agentDeploy := getAgentDeployment(manifests) + assert.NotNil(t, agentDeploy) + envCount := 0 + for _, container := range agentDeploy.Spec.Template.Spec.Containers { + if container.Name == "proxy-agent" { + envCount = len(container.Env) + } + } + assert.Equal(t, 4, envCount) + caSecret := getCASecret(manifests) + assert.NotNil(t, caSecret) + caCrt := string(caSecret.Data["ca.crt"]) + count := strings.Count(caCrt, "-----BEGIN CERTIFICATE-----") + assert.Equal(t, 2, count) + }, + }, } for _, c := range cases { @@ -880,6 +928,29 @@ func newAddOnDeploymentConfigWithCustomizedServiceDomain(name, namespace, servic } } +var fakeCA = "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM2VENDQWRFQ0ZHSG5lTUpBQ1NjR2lRSnA2K1RYa0NKRVBTVitNQTBHQ1NxR1NJYjNEUUVCQ3dVQU1ERXgKRmpBVUJnTlZCQW9NRFU5d1pXNVRhR2xtZENCQlEwMHhGekFWQmdOVkJBTU1EbmQzZHk1eVpXUm9ZWFF1WTI5dApNQjRYRFRJek1URXhNakV5TURZME4xb1hEVEkwTVRFeE1URXlNRFkwTjFvd01URVdNQlFHQTFVRUNnd05UM0JsCmJsTm9hV1owSUVGRFRURVhNQlVHQTFVRUF3d09kM2QzTG5KbFpHaGhkQzVqYjIwd2dnRWlNQTBHQ1NxR1NJYjMKRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFEUXZMbHFjYXpYZmxXNXgzcVFDSE52ZjNqTFNCY0QrY3pCczFoMApUV0p2TWEvWVd2T2MrK3VNWXg2OW1RaXRCWEFaMEsyUVpQa1BYK2lEc244Mk9mNklYTUpUSVpmZk1Wb3g4UmtqCkNlQ00vdlNaMzExVGlwa0NkaGVTbnp0WElhek1hN0ZZS3BVT2htYTF3L2RReFcvcnIwandwRG9TMFUvN0xhWGwKNHF2bUF4Wk1iSHVWaFk2S0RZSGJ2MEdKYWdqekJtVkpieTZlMFg3MkozL05ZME1KT2plYklrOTEydjBXZ1pUKwo3UWU0a29scVY1MkQvaUhYV0xFUzhXMWQrMFZUbnlRaFAzY3RvNWp3TFZyWnQ2NDFZL0lRc2ZNQ0w1bGdhVTF0Cm9UMlcvQ3F1amw5aCt0UCt2SG1rNk5JZXk2RUNIdm1MV0xLbU5nblp2M0d0bVdnZEFnTUJBQUV3RFFZSktvWkkKaHZjTkFRRUxCUUFEZ2dFQkFKSjBnd0UxSUR4SlNzaUd1TGxDMlVGV2J3U0RHMUVEK3VlQWYvRDRlV0VSWFZDUAo4aVdZZC9RckdsakYxNGxvZllHb280Vk5PL28xQWJQS2gveXB4UW16REdrVE1NaGg2WFg1bExob3RZWHZERlM2CmlkQXk5TFpiWDFUQnV5UEcwNmorbkI4eEtEY3F4aFNLYTlNb0trck9XcmtGbnFZS2syQzIyZGRvZVlZdlRjR2cKK2JmZ3RSWFJRUFdQRmt2NDR5MGlMZVh0S0VMbHBQMkMyQW5JQkU4b2hzY0JiYnloVmptem5YS1dFSTg3T0xmUgoxNDJBOWoydlVVQW80T0o5d1JCei8raDFXUXkyL3prclVUMW90MFdienY1cy91YmlUQkRpSjlQQ0k4YkZmZXplCnpDbCthbEE5aUFJdGt4OVdZS2pzaDFuVHEzTnJwVWM0MXBJWlFBQT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=" + +func newAddOnDeploymentConfigWithProxy(name, namespace string) *addonv1alpha1.AddOnDeploymentConfig { + return &addonv1alpha1.AddOnDeploymentConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: addonv1alpha1.AddOnDeploymentConfigSpec{ + NodePlacement: &addonv1alpha1.NodePlacement{ + Tolerations: tolerations, + NodeSelector: nodeSelector, + }, + ProxyConfig: addonv1alpha1.ProxyConfig{ + HTTPProxy: "http://192.168.1.1", + HTTPSProxy: "https://192.168.1.1", + CABundle: []byte(fakeCA), + NoProxy: "localhost", + }, + }, + } +} + func newLoadBalancerService(ingress string) *corev1.Service { svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -950,3 +1021,16 @@ func getProxyServerHost(deploy *appsv1.Deployment) string { } return "" } + +func getCASecret(manifests []runtime.Object) *corev1.Secret { + for _, manifest := range manifests { + switch obj := manifest.(type) { + case *corev1.Secret: + if obj.Name == "cluster-proxy-ca" { + return obj + } + } + } + + return nil +} diff --git a/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/addon-agent-deployment.yaml b/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/addon-agent-deployment.yaml index 6332d520..d392fec7 100644 --- a/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/addon-agent-deployment.yaml +++ b/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/addon-agent-deployment.yaml @@ -59,6 +59,21 @@ spec: privileged: false runAsNonRoot: true readOnlyRootFilesystem: true + env: + {{- if .Values.proxyConfig.HTTP_PROXY }} + - name: HTTP_PROXY + value: {{ .Values.proxyConfig.HTTP_PROXY }} + {{- end }} + {{- if .Values.proxyConfig.HTTPS_PROXY }} + - name: HTTPS_PROXY + value: {{ .Values.proxyConfig.HTTPS_PROXY }} + - name: ROOT_CA_CERT + value: "/etc/ca/ca.crt" + {{- end }} + {{- if .Values.proxyConfig.NO_PROXY }} + - name: NO_PROXY + value: {{ .Values.proxyConfig.NO_PROXY }} + {{- end }} volumeMounts: - name: ca mountPath: /etc/ca diff --git a/pkg/proxyagent/agent/manifests/charts/addon-agent/values.yaml b/pkg/proxyagent/agent/manifests/charts/addon-agent/values.yaml index 6d896aa8..b80ecc11 100644 --- a/pkg/proxyagent/agent/manifests/charts/addon-agent/values.yaml +++ b/pkg/proxyagent/agent/manifests/charts/addon-agent/values.yaml @@ -33,3 +33,7 @@ serviceDomain: "" tolerations: [] nodeSelector: {} +proxyConfig: + HTTP_PROXY: null + HTTPS_PROXY: null + NO_PROXY: null