diff --git a/cmd/addon-manager/main.go b/cmd/addon-manager/main.go index 531bd4cf..55fff788 100644 --- a/cmd/addon-manager/main.go +++ b/cmd/addon-manager/main.go @@ -158,11 +158,27 @@ func main() { setupLog.Error(err, "unable to set up ready check") os.Exit(1) } - addonManager.AddAgent(agent.NewProxyAgent( + caCertData, caKeyData, err := selfSigner.CA().Config.GetPEMBytes() + if err != nil { + setupLog.Error(err, "unable to get content from signer CA") + os.Exit(1) + } + + clusterProxyAddon, err := agent.NewAgentAddon( + mgr.GetScheme(), + caCertData, + caKeyData, mgr.GetClient(), - nativeClient, - selfSigner, - )) + nativeClient) + if err != nil { + setupLog.Error(err, "unable to instantiate cluster-proxy addon") + os.Exit(1) + } + + if err := addonManager.AddAgent(clusterProxyAddon); err != nil { + setupLog.Error(err, "unable to register cluster-proxy addon") + os.Exit(1) + } ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler()) defer cancel() diff --git a/e2e/certificate/certificate.go b/e2e/certificate/certificate.go index dc13b9b8..45e6d95d 100644 --- a/e2e/certificate/certificate.go +++ b/e2e/certificate/certificate.go @@ -75,6 +75,7 @@ var _ = Describe("Certificate rotation Test", return true, nil }). WithTimeout(time.Minute). + WithPolling(time.Second * 10). Should(BeTrue()) }) diff --git a/e2e/install/install.go b/e2e/install/install.go index 4c631349..58eaa98a 100644 --- a/e2e/install/install.go +++ b/e2e/install/install.go @@ -84,37 +84,7 @@ var _ = Describe("Basic install Test", It("Probe cluster health", func() { - cfg := f.HubRESTConfig() - c := f.HubRuntimeClient() - proxyConfiguration := &proxyv1alpha1.ManagedProxyConfiguration{} - err := c.Get(context.TODO(), types.NamespacedName{ - Name: "cluster-proxy", - }, proxyConfiguration) - Expect(err).NotTo(HaveOccurred()) - waitAgentReady(proxyConfiguration, f.HubNativeClient()) - - By("Running local port-forward stream to proxy service") - localProxy := util.NewRoundRobinLocalProxy( - cfg, - &atomic.Value{}, - proxyConfiguration.Spec.ProxyServer.Namespace, - common.LabelKeyComponentName+"="+common.ComponentNameProxyServer, // TODO: configurable label selector? - 8090, - ) - - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - - closeFn, err := localProxy.Listen(ctx) - Expect(err).NotTo(HaveOccurred()) - defer closeFn() - - mungledRestConfig := buildTunnelRestConfig(ctx, f, proxyConfiguration) - nativeClient, err := kubernetes.NewForConfig(mungledRestConfig) - Expect(err).NotTo(HaveOccurred()) - data, err := nativeClient.RESTClient().Get().AbsPath("/healthz").DoRaw(context.TODO()) - Expect(err).NotTo(HaveOccurred()) - Expect(string(data)).To(Equal("ok")) + probeOnce(f) }) It("ClusterProxy configuration - scale proxy agent to 1", func() { @@ -223,7 +193,6 @@ var _ = Describe("Basic install Test", }). WithTimeout(time.Minute). Should(BeTrue()) - waitAgentReady(proxyConfiguration, f.HubNativeClient()) }) @@ -251,44 +220,51 @@ var _ = Describe("Basic install Test", Expect(err).NotTo(HaveOccurred()) Expect(proxyAgentDeploy.Annotations[common.AnnotationKeyConfigurationGeneration]). To(Equal(strconv.Itoa(int(expectedGeneration)))) + waitAgentReady(proxyConfiguration, f.HubNativeClient()) }) It("Probe cluster health should work after proxy servers restart", func() { - cfg := f.HubRESTConfig() - c := f.HubRuntimeClient() - proxyConfiguration := &proxyv1alpha1.ManagedProxyConfiguration{} - - err := c.Get(context.TODO(), types.NamespacedName{ - Name: "cluster-proxy", - }, proxyConfiguration) - Expect(err).NotTo(HaveOccurred()) + probeOnce(f) + }) + }) - By("Running local port-forward stream to proxy service") - localProxy := util.NewRoundRobinLocalProxy( - cfg, - &atomic.Value{}, - proxyConfiguration.Spec.ProxyServer.Namespace, - common.LabelKeyComponentName+"="+common.ComponentNameProxyServer, // TODO: configurable label selector? - 8090, - ) +func probeOnce(f framework.Framework) { + Eventually( + func() (bool, error) { + cfg := f.HubRESTConfig() + c := f.HubRuntimeClient() + proxyConfiguration := &proxyv1alpha1.ManagedProxyConfiguration{} + err := c.Get(context.TODO(), types.NamespacedName{ + Name: "cluster-proxy", + }, proxyConfiguration) + Expect(err).NotTo(HaveOccurred()) - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() + By("Running local port-forward stream to proxy service") + localProxy := util.NewRoundRobinLocalProxyWithReqId( + cfg, + &atomic.Value{}, + proxyConfiguration.Spec.ProxyServer.Namespace, + common.LabelKeyComponentName+"="+common.ComponentNameProxyServer, // TODO: configurable label selector? + 8090, + 200, + ) - closeFn, err := localProxy.Listen(ctx) - Expect(err).NotTo(HaveOccurred()) - defer closeFn() + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() - mungledRestConfig := buildTunnelRestConfig(ctx, f, proxyConfiguration) - nativeClient, err := kubernetes.NewForConfig(mungledRestConfig) - Expect(err).NotTo(HaveOccurred()) - data, err := nativeClient.RESTClient().Get().AbsPath("/healthz").DoRaw(context.TODO()) - Expect(err).NotTo(HaveOccurred()) - Expect(string(data)).To(Equal("ok")) - }) - }) + closeFn, err := localProxy.Listen(ctx) + Expect(err).NotTo(HaveOccurred()) + defer closeFn() + mungledRestConfig := buildTunnelRestConfig(ctx, f, proxyConfiguration) + mungledRestConfig.Timeout = time.Second * 10 + nativeClient, err := kubernetes.NewForConfig(mungledRestConfig) + Expect(err).NotTo(HaveOccurred()) + data, err := nativeClient.RESTClient().Get().AbsPath("/healthz").DoRaw(context.TODO()) + return string(data) == "ok", err + }).WithTimeout(time.Minute).WithPolling(time.Second * 10) +} func buildTunnelRestConfig(ctx context.Context, f framework.Framework, proxyConfiguration *proxyv1alpha1.ManagedProxyConfiguration) *rest.Config { hubRestConfig := f.HubRESTConfig() tunnelTlsCfg, err := util.GetKonnectivityTLSConfig(hubRestConfig, proxyConfiguration) diff --git a/go.mod b/go.mod index 8da56a0f..5f6ea76e 100644 --- a/go.mod +++ b/go.mod @@ -29,17 +29,24 @@ require ( github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.1.1 // indirect + github.com/Masterminds/sprig/v3 v3.2.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cyphar/filepath-securejoin v0.2.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful v2.9.5+incompatible // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/fatih/structs v1.1.0 // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-logr/logr v1.2.0 // indirect github.com/go-logr/zapr v1.2.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -47,9 +54,12 @@ require ( github.com/google/gofuzz v1.1.0 // indirect github.com/google/uuid v1.2.0 // indirect github.com/googleapis/gnostic v0.5.5 // indirect + github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/mitchellh/copystructure v1.1.1 // indirect + github.com/mitchellh/reflectwalk v1.0.1 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -61,7 +71,12 @@ require ( github.com/prometheus/common v0.28.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/robfig/cron v1.2.0 // indirect + github.com/shopspring/decimal v1.2.0 // indirect + github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect @@ -76,11 +91,11 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 // indirect google.golang.org/protobuf v1.27.1 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + helm.sh/helm/v3 v3.7.2 // indirect k8s.io/apiextensions-apiserver v0.23.0 // indirect k8s.io/apiserver v0.23.0 // indirect k8s.io/component-base v0.23.0 // indirect diff --git a/go.sum b/go.sum index f2aa538b..ce0958e2 100644 --- a/go.sum +++ b/go.sum @@ -71,15 +71,21 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= +github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= github.com/Masterminds/squirrel v1.5.2/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/Masterminds/vcs v1.13.1/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= @@ -292,6 +298,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= @@ -359,6 +366,7 @@ github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQL github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -451,6 +459,7 @@ github.com/gobuffalo/flect v0.2.0/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7 github.com/gobuffalo/logger v1.0.3/go.mod h1:SoeejUwldiS7ZsyCBphOGURmWdwUFXs0J7TCjEhjKxM= github.com/gobuffalo/packd v1.0.0/go.mod h1:6VTc4htmJRFB7u1m/4LeMTWjFoYrUiBkU9Fdec9hrhI= github.com/gobuffalo/packr/v2 v2.8.1/go.mod h1:c/PLlOuTU+p3SybaJATW3H6lX/iK7xEz5OeMf+NnJpg= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= @@ -607,6 +616,7 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -704,6 +714,7 @@ github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.2/go.mod h1:6iaV0fGdElS6dPBx0EApTxHrcWvmJphyh2n8YBLPPZ4= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/copystructure v1.1.1 h1:Bp6x9R1Wn16SIz3OfeDr0b7RnCG2OB66Y7PQyC/cvq4= github.com/mitchellh/copystructure v1.1.1/go.mod h1:EBArHfARyrSWO/+Wyr9zwEkc6XMFB9XyNgFNmRkZZU4= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -716,6 +727,7 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= @@ -872,6 +884,7 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= @@ -892,6 +905,7 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= @@ -951,9 +965,12 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= @@ -1548,6 +1565,7 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +helm.sh/helm/v3 v3.7.2 h1:xn1OxcZEpgKpp4CCpPz1KKUyb9gAtTouXV2E3S8ChYQ= helm.sh/helm/v3 v3.7.2/go.mod h1:UXuiAn0+FfBpqbiMuwWt8/aAKkfJvnWLBJ6f4HcFs0M= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/apis/proxy/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/proxy/v1alpha1/zz_generated.deepcopy.go index 1efb8227..fc4f8b64 100644 --- a/pkg/apis/proxy/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/proxy/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated // Code generated by controller-gen. DO NOT EDIT. diff --git a/pkg/config/agent.go b/pkg/config/agent.go index f645d9c6..4bcb5774 100644 --- a/pkg/config/agent.go +++ b/pkg/config/agent.go @@ -1,6 +1,9 @@ package config -import "fmt" +import ( + "fmt" + "strings" +) var AgentImageName string @@ -10,3 +13,13 @@ func ValidateAgentImage() error { } return nil } + +func GetParsedAgentImage() (string, string, string) { + parts := strings.Split(AgentImageName, ":") + tag := "latest" + if len(parts) >= 2 { + tag = parts[len(parts)-1] + } + imgParts := strings.Split(parts[0], "/") + return strings.Join(imgParts[0:len(imgParts)-1], "/"), imgParts[len(imgParts)-1], tag +} diff --git a/pkg/proxyagent/agent/agent.go b/pkg/proxyagent/agent/agent.go index ec61732a..ab3e9e89 100644 --- a/pkg/proxyagent/agent/agent.go +++ b/pkg/proxyagent/agent/agent.go @@ -2,20 +2,13 @@ package agent import ( "context" + "embed" + "encoding/base64" "fmt" "strconv" "time" - "open-cluster-management.io/addon-framework/pkg/agent" - "open-cluster-management.io/addon-framework/pkg/utils" - addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" - clusterv1 "open-cluster-management.io/api/cluster/v1" - proxyv1alpha1 "open-cluster-management.io/cluster-proxy/pkg/apis/proxy/v1alpha1" - "open-cluster-management.io/cluster-proxy/pkg/common" - "open-cluster-management.io/cluster-proxy/pkg/proxyserver/operator/authentication/selfsigned" - "github.com/pkg/errors" - appsv1 "k8s.io/api/apps/v1" certificatesv1 "k8s.io/api/certificates/v1" csrv1 "k8s.io/api/certificates/v1" corev1 "k8s.io/api/core/v1" @@ -24,83 +17,29 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" "k8s.io/utils/pointer" + "open-cluster-management.io/addon-framework/pkg/addonfactory" + "open-cluster-management.io/addon-framework/pkg/agent" + "open-cluster-management.io/addon-framework/pkg/utils" + addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" + clusterv1 "open-cluster-management.io/api/cluster/v1" + proxyv1alpha1 "open-cluster-management.io/cluster-proxy/pkg/apis/proxy/v1alpha1" + "open-cluster-management.io/cluster-proxy/pkg/common" + "open-cluster-management.io/cluster-proxy/pkg/config" "sigs.k8s.io/controller-runtime/pkg/client" ) -var _ agent.AgentAddon = &proxyAgent{} +//go:embed manifests +var FS embed.FS const ( ProxyAgentSignerName = "open-cluster-management.io/proxy-agent-signer" ) -func NewProxyAgent(runtimeClient client.Client, nativeClient kubernetes.Interface, signer selfsigned.SelfSigner) agent.AgentAddon { - return &proxyAgent{ - runtimeClient: runtimeClient, - nativeClient: nativeClient, - selfSigner: signer, - } -} - -type proxyAgent struct { - runtimeClient client.Reader - nativeClient kubernetes.Interface - selfSigner selfsigned.SelfSigner -} - -func (p *proxyAgent) Manifests(managedCluster *clusterv1.ManagedCluster, addon *addonv1alpha1.ManagedClusterAddOn) ([]runtime.Object, error) { - // prepping - clusterAddon := &addonv1alpha1.ClusterManagementAddOn{} - if err := p.runtimeClient.Get(context.TODO(), types.NamespacedName{ - Name: addon.Name, - }, clusterAddon); err != nil { - return nil, err - } - proxyConfig := &proxyv1alpha1.ManagedProxyConfiguration{} - if err := p.runtimeClient.Get(context.TODO(), types.NamespacedName{ - Name: clusterAddon.Spec.AddOnConfiguration.CRName, - }, proxyConfig); err != nil { - return nil, err - } - - // find the referenced proxy load-balancer prescribed in the proxy config if there's any - var proxyServerLoadBalancer *corev1.Service - if proxyConfig.Spec.ProxyServer.Entrypoint.Type == proxyv1alpha1.EntryPointTypeLoadBalancerService { - entrySvc, err := p.nativeClient.CoreV1(). - Services(proxyConfig.Spec.ProxyServer.Namespace). - Get(context.TODO(), - proxyConfig.Spec.ProxyServer.Entrypoint.LoadBalancerService.Name, - metav1.GetOptions{}) - if err != nil { - return nil, errors.Wrapf(err, "failed getting proxy loadbalancer") - } - if len(entrySvc.Status.LoadBalancer.Ingress) == 0 { - return nil, fmt.Errorf("the load-balancer service for proxy-server ingress is not yet provisioned") - } - proxyServerLoadBalancer = entrySvc - } - - objs := []runtime.Object{ - newNamespace(addon.Spec.InstallNamespace), - newCASecret(addon.Spec.InstallNamespace, AgentCASecretName, p.selfSigner.CAData()), - newClusterService(addon.Spec.InstallNamespace, managedCluster.Name), - newAgentDeployment(managedCluster.Name, addon.Spec.InstallNamespace, proxyConfig, proxyServerLoadBalancer), - } - return objs, nil -} - -func (p *proxyAgent) GetAgentAddonOptions() agent.AgentAddonOptions { - caCertData, caKeyData, err := p.selfSigner.CA().Config.GetPEMBytes() - if err != nil { - klog.Fatal(err) - } - return agent.AgentAddonOptions{ - AddonName: common.AddonName, - InstallStrategy: agent.InstallAllStrategy(common.AddonInstallNamespace), - Registration: &agent.RegistrationOption{ +func NewAgentAddon(scheme *runtime.Scheme, caCertData, caKeyData []byte, runtimeClient client.Client, nativeClient kubernetes.Interface) (agent.AgentAddon, error) { + return addonfactory.NewAgentAddonFactory(common.AddonName, FS, "manifests/charts/addon-agent"). + WithAgentRegistrationOption(&agent.RegistrationOption{ CSRConfigurations: func(cluster *clusterv1.ManagedCluster) []addonv1alpha1.RegistrationConfig { return []addonv1alpha1.RegistrationConfig{ { @@ -126,9 +65,88 @@ func (p *proxyAgent) GetAgentAddonOptions() agent.AgentAddonOptions { CSRApproveCheck: func(cluster *clusterv1.ManagedCluster, addon *addonv1alpha1.ManagedClusterAddOn, csr *csrv1.CertificateSigningRequest) bool { return cluster.Spec.HubAcceptsClient }, - PermissionConfig: p.setupPermission, + PermissionConfig: NewClusterProxySetupPermissionFunc(runtimeClient, nativeClient), CSRSign: CustomSignerWithExpiry(ProxyAgentSignerName, caKeyData, caCertData, time.Hour*24*180), - }, + }). + WithInstallStrategy(agent.InstallAllStrategy(common.AddonInstallNamespace)). + WithGetValuesFuncs(GetClusterProxyValueFunc(runtimeClient, nativeClient, caCertData)). + BuildHelmAgentAddon() + +} + +func GetClusterProxyValueFunc(runtimeClient client.Client, nativeClient kubernetes.Interface, caCertData []byte) addonfactory.GetValuesFunc { + return func(cluster *clusterv1.ManagedCluster, + addon *addonv1alpha1.ManagedClusterAddOn) (addonfactory.Values, error) { + // prepping + clusterAddon := &addonv1alpha1.ClusterManagementAddOn{} + if err := runtimeClient.Get(context.TODO(), types.NamespacedName{ + Name: common.AddonName, + }, clusterAddon); err != nil { + return nil, err + } + proxyConfig := &proxyv1alpha1.ManagedProxyConfiguration{} + if err := runtimeClient.Get(context.TODO(), types.NamespacedName{ + Name: clusterAddon.Spec.AddOnConfiguration.CRName, + }, proxyConfig); err != nil { + return nil, err + } + // this is how we set the right ingress endpoint for proxy servers to + // receive handshakes from proxy agents: + // 1. upon "Hostname" type, use the prescribed hostname directly + // 2. upon "LoadBalancerService" type, use the first entry in the ip lists + // 3. otherwise defaulted to the in-cluster service endpoint + serviceEntryPoint := proxyConfig.Spec.ProxyServer.InClusterServiceName + "." + proxyConfig.Spec.ProxyServer.Namespace + // find the referenced proxy load-balancer prescribed in the proxy config if there's any + var proxyServerLoadBalancer *corev1.Service + if proxyConfig.Spec.ProxyServer.Entrypoint.Type == proxyv1alpha1.EntryPointTypeLoadBalancerService { + entrySvc, err := nativeClient.CoreV1(). + Services(proxyConfig.Spec.ProxyServer.Namespace). + Get(context.TODO(), + proxyConfig.Spec.ProxyServer.Entrypoint.LoadBalancerService.Name, + metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "failed getting proxy loadbalancer") + } + if len(entrySvc.Status.LoadBalancer.Ingress) == 0 { + return nil, fmt.Errorf("the load-balancer service for proxy-server ingress is not yet provisioned") + } + proxyServerLoadBalancer = entrySvc + } + addonAgentArgs := []string{ + "--hub-kubeconfig=/etc/kubeconfig/kubeconfig", + "--cluster-name=" + cluster.Name, + "--proxy-server-namespace=" + proxyConfig.Spec.ProxyServer.Namespace, + } + annotations := make(map[string]string) + switch proxyConfig.Spec.ProxyServer.Entrypoint.Type { + case proxyv1alpha1.EntryPointTypeHostname: + serviceEntryPoint = proxyConfig.Spec.ProxyServer.Entrypoint.Hostname.Value + case proxyv1alpha1.EntryPointTypeLoadBalancerService: + serviceEntryPoint = proxyServerLoadBalancer.Status.LoadBalancer.Ingress[0].IP + case proxyv1alpha1.EntryPointTypePortForward: + serviceEntryPoint = "127.0.0.1" + addonAgentArgs = append(addonAgentArgs, + "--enable-port-forward-proxy=true") + annotations[common.AnnotationKeyConfigurationGeneration] = strconv.Itoa(int(proxyConfig.Generation)) + } + + registry, image, tag := config.GetParsedAgentImage() + return map[string]interface{}{ + "agentDeploymentName": "cluster-proxy-proxy-agent", + "serviceDomain": "svc.cluster.local", + "includeNamespaceCreation": true, + "spokeAddonNamespace": "open-cluster-management-cluster-proxy", + + "clusterName": cluster.Name, + "registry": registry, + "image": image, + "tag": tag, + "replicas": proxyConfig.Spec.ProxyAgent.Replicas, + "base64EncodedCAData": base64.StdEncoding.EncodeToString(caCertData), + "serviceEntryPoint": serviceEntryPoint, + "agentDeploymentAnnotations": annotations, + "additionalProxyAgentArgs": addonAgentArgs, + }, nil } } @@ -141,90 +159,96 @@ func CustomSignerWithExpiry(customSignerName string, caKey, caData []byte, durat } } -func (p *proxyAgent) setupPermission(cluster *clusterv1.ManagedCluster, addon *addonv1alpha1.ManagedClusterAddOn) error { - // prepping - clusterAddon := &addonv1alpha1.ClusterManagementAddOn{} - if err := p.runtimeClient.Get(context.TODO(), types.NamespacedName{ - Name: addon.Name, - }, clusterAddon); err != nil { - return err - } - proxyConfig := &proxyv1alpha1.ManagedProxyConfiguration{} - if err := p.runtimeClient.Get(context.TODO(), types.NamespacedName{ - Name: clusterAddon.Spec.AddOnConfiguration.CRName, - }, proxyConfig); err != nil { - return err - } +type SetupPermissionFunc func(cluster *clusterv1.ManagedCluster, addon *addonv1alpha1.ManagedClusterAddOn) error - namespace := cluster.Name +func NewClusterProxySetupPermissionFunc( + runtimeClient client.Client, + nativeClient kubernetes.Interface) SetupPermissionFunc { + return func(cluster *clusterv1.ManagedCluster, addon *addonv1alpha1.ManagedClusterAddOn) error { + // prepping + clusterAddon := &addonv1alpha1.ClusterManagementAddOn{} + if err := runtimeClient.Get(context.TODO(), types.NamespacedName{ + Name: addon.Name, + }, clusterAddon); err != nil { + return err + } + proxyConfig := &proxyv1alpha1.ManagedProxyConfiguration{} + if err := runtimeClient.Get(context.TODO(), types.NamespacedName{ + Name: clusterAddon.Spec.AddOnConfiguration.CRName, + }, proxyConfig); err != nil { + return err + } - // TODO: consider switching to SSA at some point - role := &rbacv1.Role{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: "cluster-proxy-addon-agent", - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: addonv1alpha1.GroupVersion.String(), - Kind: "ManagedClusterAddOn", - Name: addon.Name, - BlockOwnerDeletion: pointer.Bool(true), - UID: addon.UID, + namespace := cluster.Name + + // TODO: consider switching to SSA at some point + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "cluster-proxy-addon-agent", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: addonv1alpha1.GroupVersion.String(), + Kind: "ManagedClusterAddOn", + Name: addon.Name, + BlockOwnerDeletion: pointer.Bool(true), + UID: addon.UID, + }, }, }, - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"coordination.k8s.io"}, - Verbs: []string{"*"}, - Resources: []string{"leases"}, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{"coordination.k8s.io"}, + Verbs: []string{"*"}, + Resources: []string{"leases"}, + }, }, - }, - } + } - roleBinding := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: "cluster-proxy-addon-agent", - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: addonv1alpha1.GroupVersion.String(), - Kind: "ManagedClusterAddOn", - Name: addon.Name, - BlockOwnerDeletion: pointer.Bool(true), - UID: addon.UID, + roleBinding := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + Name: "cluster-proxy-addon-agent", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: addonv1alpha1.GroupVersion.String(), + Kind: "ManagedClusterAddOn", + Name: addon.Name, + BlockOwnerDeletion: pointer.Bool(true), + UID: addon.UID, + }, }, }, - }, - RoleRef: rbacv1.RoleRef{ - Kind: "Role", - Name: "cluster-proxy-addon-agent", - }, - Subjects: []rbacv1.Subject{ - { - Kind: rbacv1.GroupKind, - Name: common.SubjectGroupClusterProxy, + RoleRef: rbacv1.RoleRef{ + Kind: "Role", + Name: "cluster-proxy-addon-agent", }, - }, - } + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.GroupKind, + Name: common.SubjectGroupClusterProxy, + }, + }, + } - if _, err := p.nativeClient.RbacV1().Roles(namespace).Create( - context.TODO(), - role, - metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - return err + if _, err := nativeClient.RbacV1().Roles(namespace).Create( + context.TODO(), + role, + metav1.CreateOptions{}); err != nil { + if !apierrors.IsAlreadyExists(err) { + return err + } } - } - if _, err := p.nativeClient.RbacV1().RoleBindings(namespace).Create( - context.TODO(), - roleBinding, - metav1.CreateOptions{}); err != nil { - if !apierrors.IsAlreadyExists(err) { - return err + if _, err := nativeClient.RbacV1().RoleBindings(namespace).Create( + context.TODO(), + roleBinding, + metav1.CreateOptions{}); err != nil { + if !apierrors.IsAlreadyExists(err) { + return err + } } + return nil } - return nil } const ( @@ -233,196 +257,3 @@ const ( AgentSecretName = "cluster-proxy-open-cluster-management.io-proxy-agent-signer-client-cert" AgentCASecretName = "cluster-proxy-ca" ) - -func newNamespace(targetNamespace string) *corev1.Namespace { - return &corev1.Namespace{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Namespace", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: targetNamespace, - }, - } -} - -func newAgentDeployment(clusterName, targetNamespace string, proxyConfig *proxyv1alpha1.ManagedProxyConfiguration, proxyLoadBalancer *corev1.Service) *appsv1.Deployment { - // this is how we set the right ingress endpoint for proxy servers to - // receive handshakes from proxy agents: - // 1. upon "Hostname" type, use the prescribed hostname directly - // 2. upon "LoadBalancerService" type, use the first entry in the ip lists - // 3. otherwise defaulted to the in-cluster service endpoint - serviceEntryPoint := proxyConfig.Spec.ProxyServer.InClusterServiceName + "." + proxyConfig.Spec.ProxyServer.Namespace - addonAgentArgs := []string{ - "--hub-kubeconfig=/etc/kubeconfig/kubeconfig", - "--cluster-name=" + clusterName, - "--proxy-server-namespace=" + proxyConfig.Spec.ProxyServer.Namespace, - } - annotations := make(map[string]string) - switch proxyConfig.Spec.ProxyServer.Entrypoint.Type { - case proxyv1alpha1.EntryPointTypeHostname: - serviceEntryPoint = proxyConfig.Spec.ProxyServer.Entrypoint.Hostname.Value - case proxyv1alpha1.EntryPointTypeLoadBalancerService: - serviceEntryPoint = proxyLoadBalancer.Status.LoadBalancer.Ingress[0].IP - case proxyv1alpha1.EntryPointTypePortForward: - serviceEntryPoint = "127.0.0.1" - addonAgentArgs = append(addonAgentArgs, - "--enable-port-forward-proxy=true") - annotations[common.AnnotationKeyConfigurationGeneration] = strconv.Itoa(int(proxyConfig.Generation)) - } - - one := intstr.FromInt(1) - return &appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "Deployment", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: proxyConfig.Name + "-" + common.ComponentNameProxyAgent, - Namespace: targetNamespace, - Annotations: annotations, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &proxyConfig.Spec.ProxyAgent.Replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - ApiserverNetworkProxyLabelAddon: common.AddonName, - common.LabelKeyComponentName: common.ComponentNameProxyAgent, - }, - }, - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.RollingUpdateDeploymentStrategyType, - RollingUpdate: &appsv1.RollingUpdateDeployment{ - MaxSurge: &one, - MaxUnavailable: &one, - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - ApiserverNetworkProxyLabelAddon: common.AddonName, - common.LabelKeyComponentName: common.ComponentNameProxyAgent, - }, - Annotations: annotations, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: common.ComponentNameProxyAgent, - Image: proxyConfig.Spec.ProxyAgent.Image, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{ - "/proxy-agent", - }, - Args: []string{ - "--proxy-server-host=" + serviceEntryPoint, - "--agent-identifiers=" + - "host=" + clusterName + "&" + - "host=" + clusterName + "." + targetNamespace + "&" + - "host=" + clusterName + "." + targetNamespace + ".svc.cluster.local", - "--ca-cert=/etc/ca/ca.crt", - "--agent-cert=/etc/tls/tls.crt", - "--agent-key=/etc/tls/tls.key", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "ca", - ReadOnly: true, - MountPath: "/etc/ca/", - }, - { - Name: "hub", - ReadOnly: true, - MountPath: "/etc/tls/", - }, - }, - }, - { - Name: "addon-agent", - Image: proxyConfig.Spec.ProxyAgent.Image, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{ - "/agent", - }, - Args: addonAgentArgs, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "hub-kubeconfig", - ReadOnly: true, - MountPath: "/etc/kubeconfig/", - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Port: intstr.FromInt(8888), - }, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "ca", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: AgentCASecretName, - }, - }, - }, - { - Name: "hub", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: AgentSecretName, - }, - }, - }, - { - Name: "hub-kubeconfig", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "cluster-proxy-hub-kubeconfig", - }, - }, - }, - }, - }, - }, - }, - } -} - -func newCASecret(namespace, name string, caData []byte) *corev1.Secret { - return &corev1.Secret{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Secret", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - }, - Data: map[string][]byte{ - selfsigned.TLSCACert: caData, - }, - } -} - -func newClusterService(namespace, name string) *corev1.Service { - const nativeKubernetesInClusterService = "kubernetes.default.svc.cluster.local" - return &corev1.Service{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Service", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - Name: name, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeExternalName, - ExternalName: nativeKubernetesInClusterService, - }, - } -} diff --git a/pkg/proxyagent/agent/manifests/charts/addon-agent/Chart.yaml b/pkg/proxyagent/agent/manifests/charts/addon-agent/Chart.yaml new file mode 100644 index 00000000..d871ef18 --- /dev/null +++ b/pkg/proxyagent/agent/manifests/charts/addon-agent/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: cluster-proxy-agent-only +description: A Helm chart for Cluster-Proxy OCM Addon (Agent Only) +type: application +version: 0.1.3 +appVersion: 1.0.0 diff --git a/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/addon-agent-deployment.yaml b/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/addon-agent-deployment.yaml new file mode 100644 index 00000000..71828afb --- /dev/null +++ b/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/addon-agent-deployment.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: {{ .Release.Namespace }} + name: {{ .Values.agentDeploymentName }} + annotations: + {{- with .Values.agentDeploymentAnnotations }} + {{ toYaml . | indent 8 }} + {{- end }} +spec: + replicas: {{ .Values.replicas }} + selector: + matchLabels: + open-cluster-management.io/addon: cluster-proxy + proxy.open-cluster-management.io/component-name: proxy-agent + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + template: + metadata: + annotations: + {{- with .Values.agentDeploymentAnnotations }} + {{ toYaml . | indent 8 }} + {{- end }} + labels: + open-cluster-management.io/addon: cluster-proxy + proxy.open-cluster-management.io/component-name: proxy-agent + spec: + serviceAccount: cluster-proxy + containers: + - name: proxy-agent + image: {{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }} + imagePullPolicy: IfNotPresent + command: + - /proxy-agent + args: + - --proxy-server-host={{ .Values.serviceEntryPoint }} + - --agent-identifiers={{ list (printf "host=%s" .Values.clusterName) (printf "host=%s.%s" .Values.clusterName .Release.Namespace) (printf "host=%s.%s.%s" .Values.clusterName .Release.Namespace .Values.serviceDomain) | join "&" }} + - --ca-cert=/etc/ca/ca.crt + - --agent-cert=/etc/tls/tls.crt + - --agent-key=/etc/tls/tls.key + volumeMounts: + - name: ca + mountPath: /etc/ca + readOnly: true + - name: hub + mountPath: /etc/tls + readOnly: true + - name: addon-agent + image: {{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }} + imagePullPolicy: IfNotPresent + command: + - /agent + args: + - --v=2 + {{- range .Values.additionalProxyAgentArgs }} + - {{ . }} + {{- end }} + volumeMounts: + - name: hub-kubeconfig + mountPath: /etc/kubeconfig/ + readOnly: true + volumes: + - name: ca + secret: + secretName: cluster-proxy-ca + - name: hub + secret: + secretName: cluster-proxy-open-cluster-management.io-proxy-agent-signer-client-cert + - name: hub-kubeconfig + secret: + secretName: cluster-proxy-hub-kubeconfig diff --git a/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/ca-secret.yaml b/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/ca-secret.yaml new file mode 100644 index 00000000..fd0a6175 --- /dev/null +++ b/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/ca-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + namespace: {{ .Release.Namespace }} + name: cluster-proxy-ca +data: + "ca.crt": {{ .Values.base64EncodedCAData }} \ No newline at end of file diff --git a/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/cluster-service.yaml b/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/cluster-service.yaml new file mode 100644 index 00000000..0727e328 --- /dev/null +++ b/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/cluster-service.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Service +metadata: + namespace: {{ .Release.Namespace }} + name: {{ .Values.clusterName }} +spec: + type: ExternalName + externalName: kubernetes.default.{{ .Values.serviceDomain }} \ No newline at end of file diff --git a/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/namespace.yaml b/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/namespace.yaml new file mode 100644 index 00000000..35d07947 --- /dev/null +++ b/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/namespace.yaml @@ -0,0 +1,8 @@ +{{ if .Values.includeNamespaceCreation }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Release.Namespace }} + annotations: + helm.sh/resource-policy: keep +{{ end }} \ No newline at end of file diff --git a/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/service-account.yaml b/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/service-account.yaml new file mode 100644 index 00000000..8f612926 --- /dev/null +++ b/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/service-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: {{ .Release.Namespace }} + name: cluster-proxy \ No newline at end of file diff --git a/pkg/proxyagent/agent/manifests/charts/addon-agent/values.yaml b/pkg/proxyagent/agent/manifests/charts/addon-agent/values.yaml new file mode 100644 index 00000000..6d1c4e40 --- /dev/null +++ b/pkg/proxyagent/agent/manifests/charts/addon-agent/values.yaml @@ -0,0 +1,29 @@ +clusterName: loopback + +agentDeploymentName: cluster-proxy-proxy-agent + +includeNamespaceCreation: false + +# Image registry +registry: quay.io/open-cluster-management + +# Image of the cluster-gateway instances +image: cluster-proxy + +# Image tag +tag: v0.1.3 + +# Number of replicas +replicas: 1 + +spokeAddonNamespace: "open-cluster-management-cluster-proxy" + +agentDeploymentAnnotations: {} + +additionalProxyAgentArgs: [] + +serviceEntryPoint: "" + +base64EncodedCAData: Zm9vCg== + +serviceDomain: "" \ No newline at end of file diff --git a/pkg/proxyserver/controllers/clustermanagementaddon_controller.go b/pkg/proxyserver/controllers/clustermanagementaddon_controller.go index 652039ee..cedde844 100644 --- a/pkg/proxyserver/controllers/clustermanagementaddon_controller.go +++ b/pkg/proxyserver/controllers/clustermanagementaddon_controller.go @@ -74,6 +74,7 @@ func RegisterClusterManagementAddonReconciler( SecretGetter: nativeClient.CoreV1(), ServiceGetter: nativeClient.CoreV1(), DeploymentGetter: nativeClient.AppsV1(), + EventRecorder: events.NewInMemoryRecorder("ClusterManagementAddonReconciler"), } return r.SetupWithManager(mgr) } diff --git a/pkg/util/portforward.go b/pkg/util/portforward.go index 4848412a..12ee20a2 100644 --- a/pkg/util/portforward.go +++ b/pkg/util/portforward.go @@ -38,12 +38,29 @@ func NewRoundRobinLocalProxy( podNamespace, podSelector string, targetPort int32) LocalProxyServer { + return NewRoundRobinLocalProxyWithReqId( + restConfig, + readiness, + podNamespace, + podSelector, + targetPort, + 0, + ) +} + +func NewRoundRobinLocalProxyWithReqId( + restConfig *rest.Config, + readiness *atomic.Value, + podNamespace, + podSelector string, + targetPort int32, + reqId int) LocalProxyServer { return &roundRobin{ restConfig: restConfig, proxyServerNamespace: podNamespace, podSelector: podSelector, targetPort: targetPort, - reqId: 0, + reqId: reqId, lock: &sync.Mutex{}, firstConnReceived: readiness, }