diff --git a/inttest/airgap/airgap_test.go b/inttest/airgap/airgap_test.go index e000233d764b..0af35c55a11b 100644 --- a/inttest/airgap/airgap_test.go +++ b/inttest/airgap/airgap_test.go @@ -21,12 +21,14 @@ import ( "strings" "testing" - "github.com/stretchr/testify/suite" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/k0sproject/k0s/inttest/common" "github.com/k0sproject/k0s/pkg/airgap" "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + + "github.com/stretchr/testify/suite" ) const k0sConfig = ` @@ -40,10 +42,11 @@ type AirgapSuite struct { } func (s *AirgapSuite) TestK0sGetsUp() { + ctx := s.Context() err := (&common.Airgap{ SSH: s.SSH, Logf: s.T().Logf, - }).LockdownMachines(s.Context(), + }).LockdownMachines(ctx, s.ControllerNode(0), s.WorkerNode(0), ) s.Require().NoError(err) @@ -55,6 +58,8 @@ func (s *AirgapSuite) TestK0sGetsUp() { kc, err := s.KubeClient(s.ControllerNode(0)) s.Require().NoError(err) + ctx = cancelOnBadPulledEvent(ctx, s.T(), kc) + err = s.WaitForNodeReady(s.WorkerNode(0), kc) s.NoError(err) @@ -62,47 +67,37 @@ func (s *AirgapSuite) TestK0sGetsUp() { s.Equal("bar", labels["k0sproject.io/foo"]) } - s.AssertSomeKubeSystemPods(kc) - - s.T().Log("waiting to see kube-router pods ready") - s.NoError(common.WaitForKubeRouterReady(s.Context(), kc), "kube-router did not start") - - // at that moment we can assume that all pods has at least started - events, err := kc.CoreV1().Events("kube-system").List(s.Context(), v1.ListOptions{ - Limit: 100, + s.Require().NoError(common.WaitForKubeRouterReady(ctx, kc), "While waiting for kube-router to become ready") + s.Require().NoError(common.WaitForCoreDNSReady(ctx, kc), "While waiting for CoreDNS to become ready") + s.Require().NoError(common.WaitForPodLogs(ctx, kc, "kube-system"), "While waiting for some pod logs") + + // At that moment we can assume that all pods have at least started + // We're interested only in image pull events + events, err := kc.CoreV1().Events("").List(ctx, metav1.ListOptions{ + FieldSelector: fields.AndSelectors( + fields.OneTermEqualSelector("involvedObject.kind", "Pod"), + fields.OneTermEqualSelector("reason", "Pulled"), + ).String(), }) s.Require().NoError(err) - imagesUsed := 0 - var pulledImagesMessages []string + for _, event := range events.Items { - if event.Source.Component == "kubelet" && event.Reason == "Pulled" { - // We're interested only in image pull events - s.T().Logf(event.Message) - if strings.Contains(event.Message, "already present on machine") { - imagesUsed++ - } - if strings.Contains(event.Message, "Pulling image") { - pulledImagesMessages = append(pulledImagesMessages, event.Message) - } - } - } - s.T().Logf("Used %d images from airgap bundle", imagesUsed) - if len(pulledImagesMessages) > 0 { - s.T().Logf("Image pulls messages") - for _, message := range pulledImagesMessages { - s.T().Logf(message) + if !strings.HasSuffix(event.Message, "already present on machine") { + s.Fail("Unexpected Pulled event", event.Message) + } else { + s.T().Log("Observed Pulled event:", event.Message) } - s.Fail("Require all images be installed from bundle") } + // Check that all the images have io.cri-containerd.pinned=pinned label - ssh, err := s.SSH(s.Context(), s.WorkerNode(0)) + ssh, err := s.SSH(ctx, s.WorkerNode(0)) s.Require().NoError(err) + defer ssh.Disconnect() for _, i := range airgap.GetImageURIs(v1beta1.DefaultClusterSpec(), true) { - output, err := ssh.ExecWithOutput(s.Context(), fmt.Sprintf(`k0s ctr i ls "name==%s"`, i)) + output, err := ssh.ExecWithOutput(ctx, fmt.Sprintf(`k0s ctr i ls "name==%s"`, i)) s.Require().NoError(err) s.Require().Contains(output, "io.cri-containerd.pinned=pinned", "expected %s image to have io.cri-containerd.pinned=pinned label", i) } - } func TestAirgapSuite(t *testing.T) { diff --git a/inttest/ap-airgap/airgap_test.go b/inttest/ap-airgap/airgap_test.go index 3ebf72a4bf7c..44d31686a46f 100644 --- a/inttest/ap-airgap/airgap_test.go +++ b/inttest/ap-airgap/airgap_test.go @@ -16,10 +16,18 @@ package airgap import ( "fmt" + "strings" "testing" + "time" + "github.com/k0sproject/k0s/pkg/airgap" + "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" apconst "github.com/k0sproject/k0s/pkg/autopilot/constant" appc "github.com/k0sproject/k0s/pkg/autopilot/controller/plans/core" + "github.com/k0sproject/k0s/pkg/constant" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" "github.com/k0sproject/k0s/inttest/common" aptest "github.com/k0sproject/k0s/inttest/common/autopilot" @@ -46,6 +54,9 @@ func (s *airgapSuite) SetupTest() { s.Require().NoError(aptest.WaitForCRDByName(ctx, cClient, "plans")) s.Require().NoError(aptest.WaitForCRDByName(ctx, cClient, "controlnodes")) + wClient, err := s.KubeClient(s.ControllerNode(0)) + s.Require().NoError(err) + // Create a worker join token workerJoinToken, err := s.GetJoinToken("worker") s.Require().NoError(err) @@ -53,21 +64,30 @@ func (s *airgapSuite) SetupTest() { // Start the workers using the join token s.Require().NoError(s.RunWorkersWithToken(workerJoinToken)) - wClient, err := s.KubeClient(s.ControllerNode(0)) - s.Require().NoError(err) - s.Require().NoError(s.WaitForNodeReady(s.WorkerNode(0), wClient)) -} -func (s *airgapSuite) TestApply() { - err := (&common.Airgap{ - SSH: s.SSH, - Logf: s.T().Logf, - }).LockdownMachines(s.Context(), - s.ControllerNode(0), s.WorkerNode(0), - ) + // Wait until all the cluster components are up. + s.Require().NoError(common.WaitForKubeRouterReady(ctx, wClient), "While waiting for kube-router to become ready") + s.Require().NoError(common.WaitForCoreDNSReady(ctx, wClient), "While waiting for CoreDNS to become ready") + s.Require().NoError(common.WaitForPodLogs(ctx, wClient, "kube-system"), "While waiting for some pod logs") + + // Check that none of the images in the airgap bundle are pinned. + // This will happen as soon as k0s imports them after the Autopilot update. + ssh, err := s.SSH(ctx, s.WorkerNode(0)) s.Require().NoError(err) + defer ssh.Disconnect() + for _, i := range airgap.GetImageURIs(v1beta1.DefaultClusterSpec(), true) { + if strings.HasPrefix(i, constant.KubePauseContainerImage+":") { + continue // The pause image is pinned by containerd itself + } + output, err := ssh.ExecWithOutput(ctx, fmt.Sprintf(`k0s ctr i ls "name==%s"`, i)) + if s.NoError(err, "Failed to check %s", i) { + s.NotContains(output, "io.cri-containerd.pinned=pinned", "%s is already pinned", i) + } + } +} +func (s *airgapSuite) TestApply() { planTemplate := ` apiVersion: autopilot.k0sproject.io/v1beta2 kind: Plan @@ -110,9 +130,23 @@ spec: - worker0 ` + ctx := s.Context() + + // The container images have already been pulled by the cluster. + // Airgapping is kind of cosmetic here. + err := (&common.Airgap{ + SSH: s.SSH, + Logf: s.T().Logf, + }).LockdownMachines(ctx, + s.ControllerNode(0), s.WorkerNode(0), + ) + s.Require().NoError(err) + manifestFile := "/tmp/happy.yaml" s.PutFileTemplate(s.ControllerNode(0), manifestFile, planTemplate, nil) + updateStart := time.Now() + out, err := s.RunCommandController(0, fmt.Sprintf("/usr/local/bin/k0s kubectl apply -f %s", manifestFile)) s.T().Logf("kubectl apply output: '%s'", out) s.Require().NoError(err) @@ -122,15 +156,52 @@ spec: s.NotEmpty(client) // The plan has enough information to perform a successful update of k0s, so wait for it. - _, err = aptest.WaitForPlanState(s.Context(), client, apconst.AutopilotName, appc.PlanCompleted) + _, err = aptest.WaitForPlanState(ctx, client, apconst.AutopilotName, appc.PlanCompleted) s.Require().NoError(err) - // We are not confirming the image importing functionality of k0s, but we can get a pretty good idea if it worked. - // Does the bundle exist on the worker, in the proper directory? lsout, err := s.RunCommandWorker(0, "ls /var/lib/k0s/images/bundle.tar") s.NoError(err) s.NotEmpty(lsout) + + // Wait until all the cluster components are up. + kc, err := s.KubeClient(s.ControllerNode(0)) + s.Require().NoError(err) + s.Require().NoError(common.WaitForKubeRouterReady(ctx, kc), "While waiting for kube-router to become ready") + s.Require().NoError(common.WaitForCoreDNSReady(ctx, kc), "While waiting for CoreDNS to become ready") + s.Require().NoError(common.WaitForPodLogs(ctx, kc, "kube-system"), "While waiting for some pod logs") + + // At that moment we can assume that all pods have at least started. + // Inspect the Pulled events if there are some unexpected image pulls. + events, err := kc.CoreV1().Events("").List(ctx, metav1.ListOptions{ + FieldSelector: fields.AndSelectors( + fields.OneTermEqualSelector("involvedObject.kind", "Pod"), + fields.OneTermEqualSelector("reason", "Pulled"), + ).String(), + }) + s.Require().NoError(err) + + for _, event := range events.Items { + if event.LastTimestamp.After(updateStart) { + if !strings.HasSuffix(event.Message, "already present on machine") { + s.Fail("Unexpected Pulled event", event.Message) + } else { + s.T().Log("Observed Pulled event:", event.Message) + } + } + } + + // Check that all the images in the airgap bundle have been pinned by k0s. + // This proves that k0s has processed the image bundle. + ssh, err := s.SSH(ctx, s.WorkerNode(0)) + s.Require().NoError(err) + defer ssh.Disconnect() + for _, i := range airgap.GetImageURIs(v1beta1.DefaultClusterSpec(), true) { + output, err := ssh.ExecWithOutput(ctx, fmt.Sprintf(`k0s ctr i ls "name==%s"`, i)) + if s.NoError(err, "Failed to check %s", i) { + s.Contains(output, "io.cri-containerd.pinned=pinned", "%s is not pinned", i) + } + } } // TestAirgapSuite sets up a suite using 3 controllers for quorum, and runs various