test(api): improve e2e tests logging and fix ci workflow (#501)

- add diff between timeouts of exec and kubectl commands; - decrease default ssh command timeout; - refactor workflow with kubernetes resource labels; - refactor WaitPhase func: add async await for list of resources; - refactor workflow with Eventually func; - fix ginkgo installation in ci. Signed-off-by: Roman Sysoev <[email protected]>
deckhouse · Nov 6, 2024 · 4f9c1ba · 4f9c1ba
1 parent cb9edf8
commit 4f9c1ba
Show file tree

Hide file tree

Showing 13 changed files with 254 additions and 197 deletions.
diff --git a/.github/workflows/e2e_main.yaml → ...hub/workflows/nightly_e2e_tests_main.yaml b/.github/workflows/e2e_main.yaml → ...hub/workflows/nightly_e2e_tests_main.yaml
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: e2e
+name: Nightly E2E tests(main)
 
 env:
   CI_COMMIT_REF_NAME: ${{ github.ref_name }}
@@ -43,6 +43,7 @@ jobs:
         uses: arduino/setup-task@v2
 
       - name: Install ginkgo
+        working-directory: ./tests/e2e/
         run: |
           echo "Install ginkgo"
           GINKGO_VERSION=$(go list -f '{{.Version}}' -m github.com/onsi/ginkgo/v2)

diff --git a/tests/e2e/complex_test.go b/tests/e2e/complex_test.go
@@ -50,7 +50,7 @@ func AssignIPToVMIP(name string) error {
 	waitOpts := kc.WaitOptions{
 		Namespace: conf.Namespace,
 		For:       jsonPath,
-		Timeout:   30,
+		Timeout:   ShortWaitDuration,
 	}
 	res := kubectl.WaitResources(kc.ResourceVMIP, waitOpts, name)
 	if res.Error() != nil {
@@ -68,39 +68,39 @@ var _ = Describe("Complex test", ginkgoutil.CommonE2ETestDecorators(), func() {
 	Context("When virtualization resources are applied:", func() {
 		It("result should be succeeded", func() {
 			res := kubectl.Kustomize(conf.TestData.ComplexTest, kc.KustomizeOptions{})
-			Expect(res.WasSuccess()).To(Equal(true), res.StdErr())
+			Expect(res.Error()).NotTo(HaveOccurred(), res.StdErr())
 		})
 	})
 
 	Context("When virtual images are applied:", func() {
 		It("checks VIs phases", func() {
 			By(fmt.Sprintf("VIs should be in %s phases", PhaseReady))
-			WaitPhase(kc.ResourceVI, PhaseReady, kc.GetOptions{
+			WaitPhaseByLabel(kc.ResourceVI, PhaseReady, kc.WaitOptions{
 				Labels:    testCaseLabel,
 				Namespace: conf.Namespace,
-				Output:    "jsonpath='{.items[*].metadata.name}'",
+				Timeout:   MaxWaitTimeout,
 			})
 		})
 	})
 
 	Context("When cluster virtual images are applied:", func() {
 		It("checks CVIs phases", func() {
 			By(fmt.Sprintf("CVIs should be in %s phases", PhaseReady))
-			WaitPhase(kc.ResourceCVI, PhaseReady, kc.GetOptions{
+			WaitPhaseByLabel(kc.ResourceCVI, PhaseReady, kc.WaitOptions{
 				Labels:    testCaseLabel,
 				Namespace: conf.Namespace,
-				Output:    "jsonpath='{.items[*].metadata.name}'",
+				Timeout:   MaxWaitTimeout,
 			})
 		})
 	})
 
 	Context("When virtual machine classes are applied:", func() {
 		It("checks VMClasses phases", func() {
 			By(fmt.Sprintf("VMClasses should be in %s phases", PhaseReady))
-			WaitPhase(kc.ResourceVMClass, PhaseReady, kc.GetOptions{
+			WaitPhaseByLabel(kc.ResourceVMClass, PhaseReady, kc.WaitOptions{
 				Labels:    testCaseLabel,
 				Namespace: conf.Namespace,
-				Output:    "jsonpath='{.items[*].metadata.name}'",
+				Timeout:   MaxWaitTimeout,
 			})
 		})
 	})
@@ -115,53 +115,53 @@ var _ = Describe("Complex test", ginkgoutil.CommonE2ETestDecorators(), func() {
 
 		It("checks VMIPs phases", func() {
 			By(fmt.Sprintf("VMIPs should be in %s phases", PhaseAttached))
-			WaitPhase(kc.ResourceVMIP, PhaseAttached, kc.GetOptions{
+			WaitPhaseByLabel(kc.ResourceVMIP, PhaseAttached, kc.WaitOptions{
 				Labels:    testCaseLabel,
 				Namespace: conf.Namespace,
-				Output:    "jsonpath='{.items[*].metadata.name}'",
+				Timeout:   MaxWaitTimeout,
 			})
 		})
 	})
 
 	Context("When virtual disks are applied:", func() {
 		It("checks VDs phases with consumers", func() {
 			By(fmt.Sprintf("VDs should be in %s phases", PhaseReady))
-			WaitPhase(kc.ResourceVD, PhaseReady, kc.GetOptions{
-				ExcludeLabels: []string{"hasNoConsumer"},
-				Labels:        testCaseLabel,
-				Namespace:     conf.Namespace,
-				Output:        "jsonpath='{.items[*].metadata.name}'",
+			WaitPhaseByLabel(kc.ResourceVD, PhaseReady, kc.WaitOptions{
+				ExcludedLabels: []string{"hasNoConsumer"},
+				Labels:         testCaseLabel,
+				Namespace:      conf.Namespace,
+				Timeout:        MaxWaitTimeout,
 			})
 		})
 
 		It("checks VDs phases with no consumers", func() {
 			By(fmt.Sprintf("VDs should be in %s phases", phaseByVolumeBindingMode))
-			WaitPhase(kc.ResourceVD, phaseByVolumeBindingMode, kc.GetOptions{
+			WaitPhaseByLabel(kc.ResourceVD, phaseByVolumeBindingMode, kc.WaitOptions{
 				Labels:    hasNoConsumerLabel,
 				Namespace: conf.Namespace,
-				Output:    "jsonpath='{.items[*].metadata.name}'",
+				Timeout:   MaxWaitTimeout,
 			})
 		})
 	})
 
 	Context("When virtual machines are applied:", func() {
 		It("checks VMs phases", func() {
 			By(fmt.Sprintf("VMs should be in %s phases", PhaseRunning))
-			WaitPhase(kc.ResourceVM, PhaseRunning, kc.GetOptions{
+			WaitPhaseByLabel(kc.ResourceVM, PhaseRunning, kc.WaitOptions{
 				Labels:    testCaseLabel,
 				Namespace: conf.Namespace,
-				Output:    "jsonpath='{.items[*].metadata.name}'",
+				Timeout:   MaxWaitTimeout,
 			})
 		})
 	})
 
 	Context("When virtual machine block device attachments are applied:", func() {
 		It("checks VMBDAs phases", func() {
 			By(fmt.Sprintf("VMBDAs should be in %s phases", PhaseAttached))
-			WaitPhase(kc.ResourceVMBDA, PhaseAttached, kc.GetOptions{
+			WaitPhaseByLabel(kc.ResourceVMBDA, PhaseAttached, kc.WaitOptions{
 				Labels:    testCaseLabel,
 				Namespace: conf.Namespace,
-				Output:    "jsonpath='{.items[*].metadata.name}'",
+				Timeout:   MaxWaitTimeout,
 			})
 		})
 	})
@@ -174,7 +174,7 @@ var _ = Describe("Complex test", ginkgoutil.CommonE2ETestDecorators(), func() {
 					Namespace: conf.Namespace,
 					Output:    "jsonpath='{.items[*].metadata.name}'",
 				})
-				Expect(res.WasSuccess()).To(Equal(true), res.StdErr())
+				Expect(res.Error()).NotTo(HaveOccurred(), res.StdErr())
 
 				vms := strings.Split(res.StdOut(), " ")
 				CheckExternalConnection(externalHost, httpStatusOk, vms...)
@@ -190,7 +190,7 @@ var _ = Describe("Complex test", ginkgoutil.CommonE2ETestDecorators(), func() {
 					Namespace: conf.Namespace,
 					Output:    "jsonpath='{.items[*].metadata.name}'",
 				})
-				Expect(res.WasSuccess()).To(Equal(true), res.StdErr())
+				Expect(res.Error()).NotTo(HaveOccurred(), res.StdErr())
 
 				vms := strings.Split(res.StdOut(), " ")
 				MigrateVirtualMachines(testCaseLabel, conf.TestData.ComplexTest, vms...)
@@ -200,16 +200,16 @@ var _ = Describe("Complex test", ginkgoutil.CommonE2ETestDecorators(), func() {
 		Context("When VMs migrations are applied:", func() {
 			It("checks VMs and KubevirtVMIMs phases", func() {
 				By(fmt.Sprintf("KubevirtVMIMs should be in %s phases", PhaseSucceeded))
-				WaitPhase(kc.ResourceKubevirtVMIM, PhaseSucceeded, kc.GetOptions{
+				WaitPhaseByLabel(kc.ResourceKubevirtVMIM, PhaseSucceeded, kc.WaitOptions{
 					Labels:    testCaseLabel,
 					Namespace: conf.Namespace,
-					Output:    "jsonpath='{.items[*].metadata.name}'",
+					Timeout:   MaxWaitTimeout,
 				})
 				By(fmt.Sprintf("VMs should be in %s phase", PhaseRunning))
-				WaitPhase(kc.ResourceVM, PhaseRunning, kc.GetOptions{
+				WaitPhaseByLabel(kc.ResourceVM, PhaseRunning, kc.WaitOptions{
 					Labels:    testCaseLabel,
 					Namespace: conf.Namespace,
-					Output:    "jsonpath='{.items[*].metadata.name}'",
+					Timeout:   MaxWaitTimeout,
 				})
 			})
 
@@ -219,7 +219,7 @@ var _ = Describe("Complex test", ginkgoutil.CommonE2ETestDecorators(), func() {
 					Namespace: conf.Namespace,
 					Output:    "jsonpath='{.items[*].metadata.name}'",
 				})
-				Expect(res.WasSuccess()).To(Equal(true), res.StdErr())
+				Expect(res.Error()).NotTo(HaveOccurred(), res.StdErr())
 
 				vms := strings.Split(res.StdOut(), " ")
 				CheckExternalConnection(externalHost, httpStatusOk, vms...)

diff --git a/tests/e2e/d8/d8.go b/tests/e2e/d8/d8.go
@@ -80,7 +80,7 @@ func NewD8Virtualization(conf D8VirtualizationConf) (*d8VirtualizationCMD, error
 }
 
 func (v d8VirtualizationCMD) SshCommand(vmName, command string, opts SshOptions) *executor.CMDResult {
-	timeout := LongTimeout
+	timeout := ShortTimeout
 	if opts.Timeout != 0 {
 		timeout = opts.Timeout
 	}

diff --git a/tests/e2e/executor/executor.go b/tests/e2e/executor/executor.go
@@ -81,7 +81,7 @@ func (e CMDExecutor) makeCMD(ctx context.Context, command string, stdout, stderr
 	return cmd
 }
 
-func mergeEnvs(curr []string, override []string) []string {
+func mergeEnvs(curr, override []string) []string {
 	envMap := make(map[string]string)
 
 	for _, currEnv := range curr {

diff --git a/tests/e2e/kubectl/kubectl.go b/tests/e2e/kubectl/kubectl.go
@@ -29,10 +29,11 @@ import (
 )
 
 const (
-	Cmd           = "kubectl"
-	ShortTimeout  = 10 * time.Second
-	MediumTimeout = 30 * time.Second
-	LongTimeout   = 60 * time.Second
+	Cmd              = "kubectl"
+	ShortTimeout     = 10 * time.Second
+	MediumTimeout    = 30 * time.Second
+	LongTimeout      = 60 * time.Second
+	ExecExtraTimeout = 20 * time.Second
 )
 
 type Resource string
@@ -48,6 +49,7 @@ type Kubectl interface {
 	Kustomize(directory string, opts KustomizeOptions) *executor.CMDResult
 	List(resource Resource, opts GetOptions) *executor.CMDResult
 	Wait(filepath string, opts WaitOptions) *executor.CMDResult
+	WaitResource(resource Resource, name string, opts WaitOptions) *executor.CMDResult
 	WaitResources(resource Resource, opts WaitOptions, name ...string) *executor.CMDResult
 	Patch(filepath string, opts PatchOptions) *executor.CMDResult
 	PatchResource(resource Resource, name string, opts PatchOptions) *executor.CMDResult
@@ -66,12 +68,13 @@ type CreateOptions struct {
 }
 
 type DeleteOptions struct {
-	Labels    map[string]string
-	Namespace string
+	ExcludedLabels []string
+	Labels         map[string]string
+	Namespace      string
 }
 
 type GetOptions struct {
-	ExcludeLabels  []string
+	ExcludedLabels []string
 	IgnoreNotFound bool
 	Labels         map[string]string
 	Namespace      string
@@ -85,9 +88,11 @@ type KustomizeOptions struct {
 }
 
 type WaitOptions struct {
-	Namespace string
-	For       string
-	Timeout   time.Duration
+	ExcludedLabels []string
+	Labels         map[string]string
+	Namespace      string
+	For            string
+	Timeout        time.Duration
 }
 
 type PatchOptions struct {
@@ -221,7 +226,7 @@ func (k KubectlCMD) Wait(filepath string, opts WaitOptions) *executor.CMDResult
 	cmd := k.waitOptions(fmt.Sprintf("%s wait -f %s", k.cmd, filepath), opts)
 	timeout := MediumTimeout
 	if opts.Timeout != 0 {
-		timeout = opts.Timeout
+		timeout = opts.Timeout + ExecExtraTimeout
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
@@ -232,7 +237,7 @@ func (k KubectlCMD) WaitResource(resource Resource, name string, opts WaitOption
 	cmd := k.waitOptions(fmt.Sprintf("%s wait %s %s", k.cmd, resource, name), opts)
 	timeout := MediumTimeout
 	if opts.Timeout != 0 {
-		timeout = opts.Timeout * time.Second
+		timeout = opts.Timeout + ExecExtraTimeout
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
@@ -243,7 +248,7 @@ func (k KubectlCMD) WaitResources(resource Resource, opts WaitOptions, names ...
 	cmd := k.waitOptions(fmt.Sprintf("%s wait %s %v", k.cmd, resource, strings.Join(names, " ")), opts)
 	timeout := MediumTimeout
 	if opts.Timeout != 0 {
-		timeout = opts.Timeout * time.Second
+		timeout = opts.Timeout + ExecExtraTimeout
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
@@ -278,28 +283,20 @@ func (k KubectlCMD) addNamespace(cmd, ns string) string {
 	return cmd
 }
 
-func (k KubectlCMD) addLabels(cmd string, labels map[string]string) string {
-	if len(labels) != 0 {
-		rawLabels := make([]string, 0, len(labels))
+func (k KubectlCMD) addLabels(cmd string, labels map[string]string, excludedLabels []string) string {
+	if len(labels) != 0 || len(excludedLabels) != 0 {
+		rawLabels := make([]string, 0, len(labels)+len(excludedLabels))
 
 		for k, v := range labels {
-			rawLabels = append(rawLabels, fmt.Sprintf("-l %s=%s", k, v))
+			rawLabels = append(rawLabels, fmt.Sprintf("%s=%s", k, v))
 		}
-		l := strings.Join(rawLabels, " ")
-		return fmt.Sprintf("%s %s", cmd, l)
-	}
-	return cmd
-}
 
-func (k KubectlCMD) excludeLabels(cmd string, labels []string) string {
-	if len(labels) != 0 {
-		rawLabels := make([]string, 0, len(labels))
-
-		for _, v := range labels {
-			rawLabels = append(rawLabels, fmt.Sprintf("-l '!%s'", v))
+		for _, v := range excludedLabels {
+			rawLabels = append(rawLabels, fmt.Sprintf("!%s", v))
 		}
-		l := strings.Join(rawLabels, " ")
-		return fmt.Sprintf("%s %s", cmd, l)
+
+		l := strings.Join(rawLabels, ",")
+		return fmt.Sprintf("%s -l '%s'", cmd, l)
 	}
 	return cmd
 }
@@ -340,24 +337,24 @@ func (k KubectlCMD) getOptions(cmd string, opts GetOptions) string {
 	cmd = k.addNamespace(cmd, opts.Namespace)
 	cmd = k.addOutput(cmd, opts.Output)
 	cmd = k.addIgnoreNotFound(cmd, opts.IgnoreNotFound)
-	cmd = k.addLabels(cmd, opts.Labels)
-	cmd = k.excludeLabels(cmd, opts.ExcludeLabels)
+	cmd = k.addLabels(cmd, opts.Labels, opts.ExcludedLabels)
 	return cmd
 }
 
 func (k KubectlCMD) deleteOptions(cmd string, opts DeleteOptions) string {
 	cmd = k.addNamespace(cmd, opts.Namespace)
-	cmd = k.addLabels(cmd, opts.Labels)
+	cmd = k.addLabels(cmd, opts.Labels, opts.ExcludedLabels)
 	return cmd
 }
 
 func (k KubectlCMD) waitOptions(cmd string, opts WaitOptions) string {
 	cmd = k.addNamespace(cmd, opts.Namespace)
+	cmd = k.addLabels(cmd, opts.Labels, opts.ExcludedLabels)
 	if opts.For != "" {
 		cmd = fmt.Sprintf("%s --for=%s", cmd, opts.For)
 	}
 	if opts.Timeout != 0 {
-		cmd = fmt.Sprintf("%s --timeout=%s", cmd, opts.Timeout*time.Second)
+		cmd = fmt.Sprintf("%s --timeout=%s", cmd, opts.Timeout)
 	}
 	return cmd
 }