From 297dd93d8f79aa338b8b42c8f5eb9ddb2489c7f7 Mon Sep 17 00:00:00 2001 From: Gaurav Mehta Date: Thu, 21 Mar 2024 17:01:46 +1100 Subject: [PATCH] added logic to lookup smee pod when deployed using local tinkerbell chart --- pkg/api/v1alpha1/common.go | 1 + pkg/controllers/cluster_controller.go | 4 +- .../cluster_tinkerbell_template_controller.go | 49 +++++++++++++++++-- pkg/controllers/setup.go | 6 +-- pkg/controllers/suite_test.go | 2 +- pkg/tink/template.go | 10 ++-- pkg/tink/template_test.go | 19 +------ 7 files changed, 59 insertions(+), 32 deletions(-) diff --git a/pkg/api/v1alpha1/common.go b/pkg/api/v1alpha1/common.go index 63450e0..7375166 100644 --- a/pkg/api/v1alpha1/common.go +++ b/pkg/api/v1alpha1/common.go @@ -26,4 +26,5 @@ const ( SeederConfig = "seeder-config" DefaultEndpointPort = 9090 DefaultSeederDeploymentService = "harvester-seeder-endpoint" + DefaultHegelDeploymentEndpointLookup = "smee" ) diff --git a/pkg/controllers/cluster_controller.go b/pkg/controllers/cluster_controller.go index fdd7e9b..135a0e2 100644 --- a/pkg/controllers/cluster_controller.go +++ b/pkg/controllers/cluster_controller.go @@ -270,13 +270,13 @@ func (r *ClusterReconciler) createTinkerbellHardware(ctx context.Context, cObj * // check to see if the service for tink-stack is ready tinkStackService := &corev1.Service{} - err := r.Get(ctx, types.NamespacedName{Name: seederv1alpha1.DefaultTinkStackService, Namespace: namespace}, tinkStackService) + err := r.Get(ctx, types.NamespacedName{Name: seederv1alpha1.DefaultTinkStackService, Namespace: deploymentNamespace}, tinkStackService) if err != nil { return fmt.Errorf("error fetching svc %s in ns %s: %v", seederv1alpha1.DefaultTinkStackService, seederv1alpha1.DefaultLocalClusterNamespace, err) } seederDeploymentService := &corev1.Service{} - err = r.Get(ctx, types.NamespacedName{Name: seederv1alpha1.DefaultSeederDeploymentService, Namespace: namespace}, seederDeploymentService) + err = r.Get(ctx, types.NamespacedName{Name: seederv1alpha1.DefaultSeederDeploymentService, Namespace: deploymentNamespace}, seederDeploymentService) if err != nil { return fmt.Errorf("error fetching svc %s in ns %s: %v", seederv1alpha1.DefaultSeederDeploymentService, seederv1alpha1.DefaultLocalClusterNamespace, err) diff --git a/pkg/controllers/cluster_tinkerbell_template_controller.go b/pkg/controllers/cluster_tinkerbell_template_controller.go index 9f6b7b0..6cd59d4 100644 --- a/pkg/controllers/cluster_tinkerbell_template_controller.go +++ b/pkg/controllers/cluster_tinkerbell_template_controller.go @@ -9,6 +9,7 @@ import ( tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -71,10 +72,12 @@ func (r *ClusterTinkerbellTemplateReconciler) createTinkerbellTemplate(ctx conte c := cObj.DeepCopy() if c.Status.Status == seederv1alpha1.ClusterNodesPatched || c.Status.Status == seederv1alpha1.ClusterTinkHardwareSubmitted || c.Status.Status == seederv1alpha1.ClusterRunning { // check to see if the service for tink-stack is ready - tinkStackService := &corev1.Service{} - err := r.Get(ctx, types.NamespacedName{Name: seederv1alpha1.DefaultTinkStackService, Namespace: namespace}, tinkStackService) + // if using an external tinkerbell stack this service should be ready as HegelEndpoint is used for serving harvester config via userdata on hardware object + // if using the custom tinkerbell chart in seeder repo, then tink-stack is not deployed but an nginx server is deployed + // to run with smee/boots, and this serves as a proxy to hegel endpoint, and we should switch to the same + hegelEndpoint, err := r.fetchHegelEndpoint(ctx) if err != nil { - return fmt.Errorf("error fetching svc %s in ns %s: %v", seederv1alpha1.DefaultTinkStackService, c.Namespace, err) + return fmt.Errorf("fetching hegel endpoint: %v", err) } seederConfig := &corev1.ConfigMap{} @@ -98,7 +101,7 @@ func (r *ClusterTinkerbellTemplateReconciler) createTinkerbellTemplate(ctx conte continue } - template, err := tink.GenerateTemplate(tinkStackService, seederConfig, inventory, c) + template, err := tink.GenerateTemplate(hegelEndpoint, seederConfig, inventory, c) if err != nil { return err } @@ -164,3 +167,41 @@ func (r *ClusterTinkerbellTemplateReconciler) SetupWithManager(mgr ctrl.Manager) })). Complete(r) } + +func (r *ClusterTinkerbellTemplateReconciler) fetchHegelEndpoint(ctx context.Context) (string, error) { + // check to see if the service for tink-stack is ready + // if using an external tinkerbell stack this service should be ready as HegelEndpoint is used for serving harvester config via userdata on hardware object + // if using the custom tinkerbell chart in seeder repo, then tink-stack is not deployed but an nginx server is deployed + // to run with smee/boots, and this serves as a proxy to hegel endpoint, and we should switch to the same + tinkStackService := &corev1.Service{} + err := r.Get(ctx, types.NamespacedName{Name: seederv1alpha1.DefaultTinkStackService, Namespace: deploymentNamespace}, tinkStackService) + + if err != nil { + if !apierrors.IsNotFound(err) { + return "", fmt.Errorf("error fetching svc %s in ns %s: %v", seederv1alpha1.DefaultTinkStackService, deploymentNamespace, err) + } + } else { + return tinkStackService.Status.LoadBalancer.Ingress[0].IP, nil + } + + // lookup smee pod address + podList := &corev1.PodList{} + ls := labels.SelectorFromSet(map[string]string{ + "app": seederv1alpha1.DefaultHegelDeploymentEndpointLookup, + "stack": "tinkerbell", + }) + err = r.List(ctx, podList, &client.ListOptions{ + LabelSelector: ls, + Namespace: deploymentNamespace, + }) + + if err != nil { + return "nil", err + } + + if len(podList.Items) == 0 { + return "", fmt.Errorf("no pods matching smee requirements found") + } + return podList.Items[0].Status.HostIP, nil + +} diff --git a/pkg/controllers/setup.go b/pkg/controllers/setup.go index 7a73c25..e795104 100644 --- a/pkg/controllers/setup.go +++ b/pkg/controllers/setup.go @@ -30,8 +30,8 @@ import ( ) var ( - scheme = runtime.NewScheme() - namespace string + scheme = runtime.NewScheme() + deploymentNamespace string //contains name of namespace where seeder is deployed ) const ( @@ -75,7 +75,7 @@ func (s *Server) Start(ctx context.Context) error { } // used by other methods to lookup tink-stack and harvester-seeder-deployment services - namespace = s.LeaderElectionNamespace + deploymentNamespace = s.LeaderElectionNamespace // create CRDs err = crd.Create(ctx, mgr.GetConfig()) diff --git a/pkg/controllers/suite_test.go b/pkg/controllers/suite_test.go index 85ee463..9f2d79d 100644 --- a/pkg/controllers/suite_test.go +++ b/pkg/controllers/suite_test.go @@ -114,7 +114,7 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) - namespace = "harvester-system" + deploymentNamespace = "harvester-system" err = createHarvesterSystemNamespace(ctx, k8sClient) Expect(err).NotTo(HaveOccurred()) err = createTinkStackService(ctx, k8sClient) diff --git a/pkg/tink/template.go b/pkg/tink/template.go index 2542e0c..0361690 100644 --- a/pkg/tink/template.go +++ b/pkg/tink/template.go @@ -19,7 +19,7 @@ const ( RebootHarvesterImageKey = "reboot-harvester-image" ) -func GenerateTemplate(svc *corev1.Service, cm *corev1.ConfigMap, i *seederv1alpha1.Inventory, c *seederv1alpha1.Cluster) (*tinkv1alpha1.Template, error) { +func GenerateTemplate(hegelEndpoint string, cm *corev1.ConfigMap, i *seederv1alpha1.Inventory, c *seederv1alpha1.Cluster) (*tinkv1alpha1.Template, error) { template := &tinkv1alpha1.Template{ ObjectMeta: metav1.ObjectMeta{ Name: i.Name, @@ -27,7 +27,7 @@ func GenerateTemplate(svc *corev1.Service, cm *corev1.ConfigMap, i *seederv1alph }, } - data, err := generateDataTemplate(svc, cm, i, c) + data, err := generateDataTemplate(hegelEndpoint, cm, i, c) if err != nil { return nil, fmt.Errorf("error generating template data: %v", err) } @@ -36,7 +36,7 @@ func GenerateTemplate(svc *corev1.Service, cm *corev1.ConfigMap, i *seederv1alph return template, nil } -func generateDataTemplate(svc *corev1.Service, cm *corev1.ConfigMap, i *seederv1alpha1.Inventory, c *seederv1alpha1.Cluster) (*string, error) { +func generateDataTemplate(hegelEndpoint string, cm *corev1.ConfigMap, i *seederv1alpha1.Inventory, c *seederv1alpha1.Cluster) (*string, error) { //set StreamHarvester action environment variables DefaultStreamHarvesterAction.Environment[DestDisk] = i.Spec.PrimaryDisk DefaultStreamHarvesterAction.Environment[ImageURL] = fmt.Sprintf("%s/%s/harvester-%s-%s.raw.gz", c.Spec.ImageURL, c.Spec.HarvesterVersion, c.Spec.HarvesterVersion, i.Spec.Arch) @@ -44,7 +44,9 @@ func generateDataTemplate(svc *corev1.Service, cm *corev1.ConfigMap, i *seederv1 //set ConfigureHarvester action environment variables DefaultConfigureHarvesterAction.Environment[HarvesterDevice] = i.Spec.PrimaryDisk DefaultConfigureHarvesterAction.Environment[HarvesterCloudInitURL] = fmt.Sprintf("http://%s:%s/2009-04-04/user-data", - svc.Status.LoadBalancer.Ingress[0].IP, HegelDefaultPort) + hegelEndpoint, HegelDefaultPort) + + //svc.Status.LoadBalancer.Ingress[0].IP // override images if specified if cm != nil { diff --git a/pkg/tink/template_test.go b/pkg/tink/template_test.go index 7c4e42c..6bee4fc 100644 --- a/pkg/tink/template_test.go +++ b/pkg/tink/template_test.go @@ -6,7 +6,6 @@ import ( seederv1alpha1 "github.com/harvester/seeder/pkg/api/v1alpha1" "github.com/rancher/wrangler/pkg/yaml" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -33,23 +32,7 @@ func Test_GenerateTemplate(t *testing.T) { }, } - tinkStackSvc := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: "tink-stack", - Namespace: "harvester-system", - }, - Status: corev1.ServiceStatus{ - LoadBalancer: corev1.LoadBalancerStatus{ - Ingress: []corev1.LoadBalancerIngress{ - { - IP: "192.168.1.100", - }, - }, - }, - }, - } - - template, err := GenerateTemplate(tinkStackSvc, nil, i, c) + template, err := GenerateTemplate("192.168.1.100", nil, i, c) assert.NoError(err, "exppected no error during template generation") assert.Equal(template.Name, i.Name, "expected template name to match inventory name") assert.Equal(template.Namespace, i.Namespace, "expected template namespace to match inventory namespace")