chore(ci): refactor e2e package and ci tests to speed up ci (#404)

* chore(ci): refactor e2e package and ci tests to speed up ci * fix: pkg definition in workflow file * chore: duplicate main_test.go file for bittwister pkg * fix: explicitly set fail-fast false * chore: move testdata to fix tests after refactor * fix(e2e/system): fix failing TestFolder * chore: skip TestBuildFromGit due to comment * chore(ci): clean up TestEnvToJSON test after refactor * feat: add pkg specific timeouts to testing ci * fix: build from git test to use knuu branch * fix: revert file perm changes due to failing test and ignore lint error
celestiaorg · May 31, 2024 · 20f3ce9 · 20f3ce9
1 parent 4e6dcf0
commit 20f3ce9
Show file tree

Hide file tree

Showing 23 changed files with 149 additions and 69 deletions.
diff --git a/.github/workflows/knuu_testing.yml b/.github/workflows/knuu_testing.yml
@@ -4,11 +4,23 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main]
   workflow_dispatch:
+  merge_group:
 
 jobs:
   test:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - pkg: "./pkg/..."
+            timeout: 10m
+          - pkg: "./e2e/basic"
+            timeout: 15m
+          - pkg: "./e2e/system"
+            timeout: 15m
+          - pkg: "./e2e/bittwister"
+            timeout: 60m
     runs-on: ubuntu-latest
 
     steps:
@@ -28,10 +40,10 @@ jobs:
           echo "${KUBECONFIG_FILE}" > $HOME/.kube/config
 
       - name: Run Tests
-        run: make test
+        run: make test pkgs=${{ matrix.pkg }} timeout=${{ matrix.timeout }}
         env:
           KNUU_SKIP_CLEANUP: "false"
-          KNUU_TIMEOUT: "240m" # needed as we run all the tests
+          KNUU_TIMEOUT: "${{ matrix.timeout }}"
           GRAFANA_ENDPOINT: ${{ secrets.GRAFANA_ENDPOINT }}
           GRAFANA_USERNAME: ${{ secrets.GRAFANA_USERNAME }}
           GRAFANA_TOKEN: ${{ secrets.GRAFANA_TOKEN }}
diff --git a/Makefile b/Makefile
@@ -1,6 +1,7 @@
 pkgs := $(shell go list ./...)
 run := .
 count := 1
+timeout := 120m
 
 ## help: Show this help message
 help: Makefile
@@ -59,5 +60,5 @@ vet:
 
 ## test: Run the testsuite
 test: vet 
-	KNUU_TIMEOUT=120m go test -v $(pkgs) -run $(run) -count=$(count) -timeout 120m
+	KNUU_TIMEOUT=120m go test -v $(pkgs) -run $(run) -count=$(count) -timeout $(timeout)
 .PHONY: test
diff --git a/e2e/basic/assert_cleanups.go → e2e/assert_cleanups.go b/e2e/basic/assert_cleanups.go → e2e/assert_cleanups.go
@@ -1,4 +1,4 @@
-package basic
+package e2e
 
 import (
 	"os"
@@ -7,8 +7,8 @@ import (
 	"github.com/celestiaorg/knuu/pkg/knuu"
 )
 
-// assertCleanupInstance is a helper function that cleans up a single instance.
-func assertCleanupInstance(t *testing.T, instance *knuu.Instance) error {
+// AssertCleanupInstance is a helper function that cleans up a single instance.
+func AssertCleanupInstance(t *testing.T, instance *knuu.Instance) error {
 	if instance == nil {
 		t.Fatal("Instance is nil")
 	}
@@ -19,8 +19,8 @@ func assertCleanupInstance(t *testing.T, instance *knuu.Instance) error {
 	return nil
 }
 
-// assertCleanupInstances is a helper function that cleans up a list of instances.
-func assertCleanupInstances(t *testing.T, executor *knuu.Executor, instances []*knuu.Instance) error {
+// AssertCleanupInstances is a helper function that cleans up a list of instances.
+func AssertCleanupInstances(t *testing.T, executor *knuu.Executor, instances []*knuu.Instance) error {
 	if os.Getenv("KNUU_SKIP_CLEANUP") == "true" {
 		t.Log("Skipping cleanup")
 		return nil

diff --git a/e2e/basic/assert_create.go → e2e/assert_create.go b/e2e/basic/assert_create.go → e2e/assert_create.go
@@ -1,4 +1,4 @@
-package basic
+package e2e
 
 import (
 	"testing"
@@ -19,8 +19,8 @@ const (
 	nginxPath = "/usr/share/nginx/html"
 )
 
-// assertCreateInstanceNginxWithVolumeOwner creates and configures an instance with common settings used across tests.
-func assertCreateInstanceNginxWithVolumeOwner(t *testing.T, instanceName string) *knuu.Instance {
+// AssertCreateInstanceNginxWithVolumeOwner creates and configures an instance with common settings used across tests but doesn't call commit.
+func AssertCreateInstanceNginxWithVolumeOwnerWithoutCommit(t *testing.T, instanceName string) *knuu.Instance {
 	instance, err := knuu.NewInstance(instanceName)
 	if err != nil {
 		t.Fatalf("Error creating instance '%v': %v", instanceName, err)
@@ -41,7 +41,13 @@ func assertCreateInstanceNginxWithVolumeOwner(t *testing.T, instanceName string)
 	if err != nil {
 		t.Fatalf("Error adding volume: %v", err)
 	}
-	err = instance.Commit()
+	return instance
+}
+
+// AssertCreateInstanceNginxWithVolumeOwner creates and configures an instance with common settings used across tests.
+func AssertCreateInstanceNginxWithVolumeOwner(t *testing.T, instanceName string) *knuu.Instance {
+	instance := AssertCreateInstanceNginxWithVolumeOwnerWithoutCommit(t, instanceName)
+	err := instance.Commit()
 	if err != nil {
 		t.Fatalf("Error committing instance '%v': %v", instanceName, err)
 	}

diff --git a/e2e/basic/.env b/e2e/basic/.env
diff --git a/e2e/basic/probe_test.go b/e2e/basic/probe_test.go
@@ -37,7 +37,7 @@ func TestProbe(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Error adding volume: %v", err)
 	}
-	err = web.AddFile("resources/html/index.html", "/usr/share/nginx/html/index.html", "0:0")
+	err = web.AddFile("../system/resources/html/index.html", "/usr/share/nginx/html/index.html", "0:0")
 	if err != nil {
 		t.Fatalf("Error adding file '%v':", err)
 	}

diff --git a/e2e/basic/bittwister_test.go → e2e/bittwister/bittwister_test.go b/e2e/basic/bittwister_test.go → e2e/bittwister/bittwister_test.go
@@ -1,4 +1,4 @@
-package basic
+package bittwister
 
 import (
 	"context"

diff --git a/e2e/bittwister/main_test.go b/e2e/bittwister/main_test.go
@@ -0,0 +1,20 @@
+package bittwister
+
+import (
+	"os"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/celestiaorg/knuu/pkg/knuu"
+)
+
+func TestMain(m *testing.M) {
+	err := knuu.Initialize()
+	if err != nil {
+		logrus.Fatalf("error initializing knuu: %v", err)
+	}
+	logrus.Infof("Scope: %s", knuu.Scope())
+	exitVal := m.Run()
+	os.Exit(exitVal)
+}
diff --git a/e2e/basic/build_from_git_test.go → e2e/system/build_from_git_test.go b/e2e/basic/build_from_git_test.go → e2e/system/build_from_git_test.go
@@ -1,9 +1,8 @@
-package basic
+package system
 
 import (
 	"context"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/require"
 
@@ -12,14 +11,11 @@ import (
 	"github.com/celestiaorg/knuu/pkg/knuu"
 )
 
-// This test is just an example to show how to
-// setup the test instance to be built from a git repo
 func TestBuildFromGit(t *testing.T) {
 	t.Parallel()
-	// Setup
 
-	ctx, cancel := context.WithTimeout(context.Background(), 45*time.Minute)
-	defer cancel()
+	// Setup
+	ctx := context.Background()
 
 	// The default image builder is kaniko here
 	kn, err := knuu.New(ctx)
@@ -30,9 +26,8 @@ func TestBuildFromGit(t *testing.T) {
 
 	// This is a blocking call which builds the image from git repo
 	err = sampleInstance.SetGitRepo(ctx, builder.GitContext{
-		Repo:   "https://github.com/celestiaorg/celestia-app.git",
-		Branch: "main",
-		// Commit:   "5ce94f4f010e366df301d25cd5d797c3147ff884",
+		Repo:     "https://github.com/celestiaorg/knuu.git",
+		Branch:   "test/build-from-git", // This branch has a Dockerfile and is protected as to not be deleted
 		Username: "",
 		Password: "",
 	})

diff --git a/e2e/basic/env_to_json_test.go → e2e/system/env_to_json_test.go b/e2e/basic/env_to_json_test.go → e2e/system/env_to_json_test.go
@@ -1,4 +1,4 @@
-package basic
+package system
 
 import (
 	"encoding/json"
@@ -15,8 +15,8 @@ import (
 
 func TestEnvToJSON(t *testing.T) {
 	t.Parallel()
-	// Setup
 
+	// Setup
 	executor, err := knuu.NewExecutor()
 	if err != nil {
 		t.Fatalf("Error creating executor: %v", err)
@@ -25,15 +25,37 @@ func TestEnvToJSON(t *testing.T) {
 	const numberOfInstances = 2
 	instances := make([]*knuu.Instance, numberOfInstances)
 
-	// get the values from the .env
-
-	test := os.Getenv("TEST")
-	test2 := os.Getenv("TEST_2")
-	test3 := os.Getenv("TEST_3")
+	// Define the env vars
+	testEnvVarKey1 := "TESTKEY1"
+	testEnvVarKey2 := "TESTKEY2"
+	testEnvVarKey3 := "TESTKEY3"
+	testEnvVarValue1 := "testvalue1"
+	testEnvVarValue2 := "testvalue2"
+	testEnvVarValue3 := "testvalue3"
+
+	// Set the OS env vars
+	setEnv := func(key, value string) {
+		err = os.Setenv(key, value)
+		if err != nil {
+			t.Fatalf("Error setting env var '%v': %v", key, err)
+		}
+	}
+	setEnv(testEnvVarKey1, testEnvVarValue1)
+	setEnv(testEnvVarKey2, testEnvVarValue2)
+	setEnv(testEnvVarKey3, testEnvVarValue3)
+
+	// Define helper function to get env vars
+	getEnv := func(key string) string {
+		value := os.Getenv(key)
+		if value == "" {
+			t.Fatalf("Error getting env var '%v'", key)
+		}
+		return value
+	}
 	jsonBytes, err := json.Marshal(map[string]string{
-		"TEST":   test,
-		"TEST_2": test2,
-		"TEST_3": test3,
+		testEnvVarKey1: getEnv(testEnvVarKey1),
+		testEnvVarKey2: getEnv(testEnvVarKey2),
+		testEnvVarKey3: getEnv(testEnvVarKey3),
 	})
 	if err != nil {
 		t.Fatalf("Error converting env vars to JSON: %v", err)
@@ -110,6 +132,7 @@ func TestEnvToJSON(t *testing.T) {
 			t.Fatalf("Error executing command: %v", err)
 		}
 
-		assert.Equal(t, "{\"TEST\":\"test\",\"TEST_2\":\"test2\",\"TEST_3\":\"test3\"}\n", wget)
+		expected := fmt.Sprintf("{\"%s\":\"%s\",\"%s\":\"%s\",\"%s\":\"%s\"}\n", testEnvVarKey1, testEnvVarValue1, testEnvVarKey2, testEnvVarValue2, testEnvVarKey3, testEnvVarValue3)
+		assert.Equal(t, expected, wget)
 	}
 }
diff --git a/e2e/basic/external_file_test.go → e2e/system/external_file_test.go b/e2e/basic/external_file_test.go → e2e/system/external_file_test.go
@@ -1,4 +1,4 @@
-package basic
+package system
 
 import (
 	"io"

diff --git a/e2e/basic/file_test.go → e2e/system/file_test.go b/e2e/basic/file_test.go → e2e/system/file_test.go
@@ -1,4 +1,4 @@
-package basic
+package system
 
 import (
 	"context"

diff --git a/e2e/basic/file_test_image_cached_test.go → e2e/system/file_test_image_cached_test.go b/e2e/basic/file_test_image_cached_test.go → e2e/system/file_test_image_cached_test.go
@@ -1,4 +1,4 @@
-package basic
+package system
 
 import (
 	"fmt"
@@ -7,6 +7,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
+	"github.com/celestiaorg/knuu/e2e"
 	"github.com/celestiaorg/knuu/pkg/knuu"
 )
 
@@ -23,7 +24,7 @@ func TestFileCached(t *testing.T) {
 
 	for i := 0; i < numberOfInstances; i++ {
 		instanceName := fmt.Sprintf("web%d", i+1)
-		instances[i] = assertCreateInstanceNginxWithVolumeOwner(t, instanceName)
+		instances[i] = e2e.AssertCreateInstanceNginxWithVolumeOwner(t, instanceName)
 	}
 
 	var wgFolders sync.WaitGroup
@@ -43,7 +44,7 @@ func TestFileCached(t *testing.T) {
 
 	t.Cleanup(func() {
 		// Cleanup
-		err := assertCleanupInstances(t, executor, instances)
+		err := e2e.AssertCleanupInstances(t, executor, instances)
 		if err != nil {
 			t.Fatalf("Error cleaning up: %v", err)
 		}

diff --git a/e2e/basic/files_to_volumes_cm_test.go → e2e/system/files_to_volumes_cm_test.go b/e2e/basic/files_to_volumes_cm_test.go → e2e/system/files_to_volumes_cm_test.go
@@ -1,4 +1,4 @@
-package basic
+package system
 
 import (
 	"fmt"
@@ -9,6 +9,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/celestiaorg/knuu/e2e"
 	"github.com/celestiaorg/knuu/pkg/knuu"
 )
 
@@ -44,7 +45,7 @@ func TestNoVolumesNoFiles(t *testing.T) {
 
 	// Cleanup
 	t.Cleanup(func() {
-		err := assertCleanupInstance(t, instance)
+		err := e2e.AssertCleanupInstance(t, instance)
 		if err != nil {
 			t.Fatalf("Error cleaning up: %v", err)
 		}
@@ -110,7 +111,7 @@ func TestOneVolumeNoFiles(t *testing.T) {
 
 	// Cleanup
 	t.Cleanup(func() {
-		err := assertCleanupInstance(t, instance)
+		err := e2e.AssertCleanupInstance(t, instance)
 		if err != nil {
 			t.Fatalf("Error cleaning up: %v", err)
 		}
@@ -156,7 +157,7 @@ func TestNoVolumesOneFile(t *testing.T) {
 
 	for i := 0; i < numberOfInstances; i++ {
 		instanceName := fmt.Sprintf("web%d", i+1)
-		instances[i] = assertCreateInstanceNginxWithVolumeOwner(t, instanceName)
+		instances[i] = e2e.AssertCreateInstanceNginxWithVolumeOwner(t, instanceName)
 	}
 
 	var wgFolders sync.WaitGroup
@@ -187,7 +188,7 @@ func TestNoVolumesOneFile(t *testing.T) {
 
 	// Cleanup
 	t.Cleanup(func() {
-		err := assertCleanupInstances(t, executor, instances)
+		err := e2e.AssertCleanupInstances(t, executor, instances)
 		if err != nil {
 			t.Fatalf("Error cleaning up: %v", err)
 		}
@@ -239,7 +240,7 @@ func TestOneVolumeOneFile(t *testing.T) {
 
 	for i := 0; i < numberOfInstances; i++ {
 		instanceName := fmt.Sprintf("web%d", i+1)
-		instances[i] = assertCreateInstanceNginxWithVolumeOwner(t, instanceName)
+		instances[i] = e2e.AssertCreateInstanceNginxWithVolumeOwner(t, instanceName)
 	}
 
 	var wgFolders sync.WaitGroup
@@ -268,7 +269,7 @@ func TestOneVolumeOneFile(t *testing.T) {
 
 	t.Cleanup(func() {
 		// Cleanup
-		err := assertCleanupInstances(t, executor, instances)
+		err := e2e.AssertCleanupInstances(t, executor, instances)
 		if err != nil {
 			t.Fatalf("Error cleaning up: %v", err)
 		}
@@ -320,7 +321,7 @@ func TestOneVolumeTwoFiles(t *testing.T) {
 
 	for i := 0; i < numberOfInstances; i++ {
 		instanceName := fmt.Sprintf("web%d", i+1)
-		instances[i] = assertCreateInstanceNginxWithVolumeOwner(t, instanceName)
+		instances[i] = e2e.AssertCreateInstanceNginxWithVolumeOwner(t, instanceName)
 	}
 
 	var wgFolders sync.WaitGroup
@@ -352,7 +353,7 @@ func TestOneVolumeTwoFiles(t *testing.T) {
 
 	t.Cleanup(func() {
 		// Cleanup
-		err := assertCleanupInstances(t, executor, instances)
+		err := e2e.AssertCleanupInstances(t, executor, instances)
 		if err != nil {
 			t.Fatalf("Error cleaning up: %v", err)
 		}