chore: add deployment scaling in load test

.github/workflows/test.yml

@@ -51,27 +51,3 @@ jobs:
             cache-
       - name: Test
         run: make test-prod
-
-  test-load:
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    steps:
-      - name: Install Go
-        uses: buildjet/setup-go@v5
-        with:
-          go-version: 1.22.x
-      - name: Setup kubernetes cluster
-        uses: container-tools/kind-action@v2
-      - name: Checkout code
-        uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
-      - uses: buildjet/cache@v4
-        with:
-          path: |
-            ~/go/pkg/mod
-            ~/.cache/go-build
-            .bin
-          key: cache-${{ hashFiles('**/go.sum') }}-${{ hashFiles('.bin/*') }}
-          restore-keys: |
-            cache-
-      - name: Test
-        run: make test-load

Makefile

@@ -63,7 +63,10 @@ test: manifests generate fmt vet envtest  ## Run tests.
 test-prod: manifests generate fmt vet envtest  ## Run tests.
 	$(MAKE) gotest-prod
 
-test-load: manifests generate fmt vet envtest  ## Run tests.
+test-load: envtest  ## Run tests.
+	kubectl delete events --all -n testns
+	kubectl delete deployments --all -n testns
+	kubectl delete pods --all -n testns
 	$(MAKE) gotest-load
 
 .PHONY: gotest
@@ -77,7 +80,7 @@ gotest-prod:
 .PHONY: gotest-load
 gotest-load:
 	make -C fixtures/load k6
-	LOAD_TEST=1 KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./tests -coverprofile cover.out
+	LOAD_TEST=1 KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -v ./tests -coverprofile cover.out
 
 .PHONY: env
 env: envtest ## Run tests.

cmd/operator.go

@@ -83,12 +83,12 @@ func run(ctx dutyContext.Context, args []string) error {
 	utilruntime.Must(v1.AddToScheme(scheme))
 
 	registerJobs(ctx, args)
-	scrapers.StartEventListener(ctx)
+	scrapers.StartEventListener(dutyCtx)
 
 	go serve(dutyCtx)
 	go tableUpdatesHandler(dutyCtx)
 
-	return launchKopper(ctx)
+	return launchKopper(dutyCtx)
 }
 
 func launchKopper(ctx dutyContext.Context) error {

fixtures/load/load.ts

@@ -131,6 +131,38 @@ const namespaceSpec = {
   }
 }
 
+const deploymentSpec = {
+  apiVersion: "apps/v1",
+  kind: "Deployment",
+  metadata: {
+    name: "nginx",
+    namespace: ns,
+  },
+  spec: {
+    replicas: 3,
+    selector: {
+      matchLabels: {
+        app: "nginx"
+      }
+    },
+    template: {
+      metadata: {
+        labels: {
+          app: "nginx"
+        }
+      },
+      spec: {
+        containers: [
+          {
+            name: "nginx",
+            image: "nginx:alpine",
+          }
+        ]
+      }
+    }
+  }
+};
+
 let count = 10
 export default function() {
   kubernetes = new Kubernetes();
@@ -171,7 +203,7 @@ export default function() {
 
     // Write this to file
     console.log(`Crashing pod: ${podName} at  ${new Date().toLocaleString()}`);
-    file.appendString('log.txt', `${podName},${new Date().toISOString()}\n`)
+    file.appendString('log.txt', `${podName},crash,${new Date().toISOString()}\n`)
 
     try {
       let response = proxyGet(kubernetes.get("Pod", podName, ns), "panic", 9898)
@@ -191,4 +223,14 @@ export default function() {
   pods.map(function(pod) {
     console.log(`  ${pod.metadata.name} ${pod.status.phase}: restarts=${pod.status.containerStatuses[0].restartCount}`);
   });
+
+  // Create deployment to scale up and down
+  console.log(`Creating nginx deployment`);
+  kubernetes.apply(JSON.stringify(deploymentSpec))
+
+  k6.sleep(5);
+  const deployment1Replica = JSON.parse(JSON.stringify(deploymentSpec));
+  deployment1Replica.spec.replicas = 1;
+  file.appendString('log.txt', `${deploymentSpec.metadata.name},scaledown,${new Date().toISOString()}\n`)
+  kubernetes.apply(JSON.stringify(deployment1Replica))
 }

tests/load_test.go

@@ -16,6 +16,8 @@ import (
 	"github.com/flanksource/duty/tests/setup"
 	ginkgo "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
 )
 
 func TestLoad(t *testing.T) {
@@ -27,10 +29,31 @@ var (
 	DefaultContext context.Context
 )
 
+type ChangeTimes []struct {
+	ChangeType string
+	CreatedAt  time.Time
+	Name       string
+	Details    string
+}
+
 var _ = ginkgo.BeforeSuite(func() {
+	logger.Infof("Set defaukt context")
 	DefaultContext = setup.BeforeSuiteFn()
 
+	kubeconfig := clientcmd.RecommendedHomeFile
+	config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+	if err != nil {
+		panic(err)
+	}
+
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		panic(err)
+	}
+	DefaultContext = DefaultContext.WithKubernetes(clientset, config)
+
 })
+
 var _ = ginkgo.AfterSuite(setup.AfterSuiteFn)
 
 var _ = ginkgo.Describe("Load Test", ginkgo.Ordered, func() {
@@ -50,6 +73,7 @@ var _ = ginkgo.Describe("Load Test", ginkgo.Ordered, func() {
 					Watch: []v1.KubernetesResourceToWatch{
 						{ApiVersion: "v1", Kind: "Namespace"},
 						{ApiVersion: "v1", Kind: "Pod"},
+						//{ApiVersion: "apps/v1", Kind: "Deployment"},
 					},
 				}},
 			},
@@ -74,38 +98,36 @@ var _ = ginkgo.Describe("Load Test", ginkgo.Ordered, func() {
 
 		os.Remove("log.txt")
 
+		time.Sleep(15 * time.Second)
+		logger.Infof("Exec k6")
 		cmd := exec.Command("../fixtures/load/k6", "run", "../fixtures/load/load.ts", "--insecure-skip-tls-verify")
 		err = cmd.Run()
 		if err != nil {
 			logger.Errorf("Error is %v", err)
 			panic(err)
 		}
 
-		time.Sleep(3 * time.Minute)
+		logger.Infof("End k6")
+		time.Sleep(2 * time.Minute)
 
 		var count int64
 		Expect(scraperCtx.DB().Table("config_changes").Count(&count).Error).To(BeNil())
 		Expect(count).ToNot(Equal(int64(0)))
 
-		var changes []struct {
-			ChangeType string
-			CreatedAt  time.Time
-			Name       string
-		}
-
+		var podinfoChanges ChangeTimes
 		err = scraperCtx.DB().Raw(`
             SELECT cc.change_type, cc.created_at, ci.name FROM config_changes cc
             INNER JOIN config_items ci ON cc.config_id = ci.id
             WHERE ci.name LIKE 'podinfo%'
-            `).Scan(&changes).Error
+            `).Scan(&podinfoChanges).Error
 
 		Expect(err).To(BeNil())
 
-		changeDiffs := make(map[string]time.Time)
-		for _, c := range changes {
+		podinfoChangeDiffs := make(map[string]time.Time)
+		for _, c := range podinfoChanges {
 			logger.Infof("Change is %v", c)
 			if c.ChangeType == v1.ChangeTypeDiff {
-				changeDiffs[c.Name] = c.CreatedAt
+				podinfoChangeDiffs[c.Name] = c.CreatedAt
 			}
 		}
 
@@ -114,21 +136,56 @@ var _ = ginkgo.Describe("Load Test", ginkgo.Ordered, func() {
 		lines := strings.Split(string(f), "\n")
 
 		k6CrashTime := make(map[string]time.Time)
+		deployTimes := make(map[string]time.Time)
 		for _, line := range lines {
 			if strings.TrimSpace(line) == "" {
 				continue
 			}
 			parts := strings.Split(line, ",")
-			t, err := time.Parse(time.RFC3339, parts[1])
+			t, err := time.Parse(time.RFC3339, parts[2])
 			Expect(err).To(BeNil())
 
-			k6CrashTime[parts[0]] = t
+			if parts[1] == "crash" {
+				k6CrashTime[parts[0]] = t
+			} else if parts[1] == "scaledown" {
+				deployTimes[parts[0]] = t
+			}
 
 			logger.Infof("N=%s t=%s", parts[0], t)
 		}
 
 		for k, v := range k6CrashTime {
-			changeLog, ok := changeDiffs[k]
+			changeLog, ok := podinfoChangeDiffs[k]
+			if !ok {
+				panic("not found " + k)
+			}
+			td := changeLog.Sub(v)
+			logger.Infof("Delta for %s is %v", k, td)
+			Expect(td).To(BeNumerically("<", time.Minute))
+		}
+
+		var nginxChanges ChangeTimes
+		err = scraperCtx.DB().Raw(`
+            SELECT cc.change_type, cc.created_at, ci.name FROM config_changes cc
+            INNER JOIN config_items ci ON cc.config_id = ci.id
+            WHERE ci.name LIKE 'nginx%'
+            ORDER BY cc.created_at ASC
+            `).Scan(&nginxChanges).Error
+
+		Expect(err).To(BeNil())
+
+		nginxChangeDiffs := make(map[string]time.Time)
+		nginxCounter := 0
+		for _, c := range nginxChanges {
+			logger.Infof("Nginx change is %v", c)
+			if c.ChangeType == "ScalingReplicaSet" && nginxCounter != 0 {
+				nginxChangeDiffs[c.Name] = c.CreatedAt
+			}
+			nginxCounter += 1
+		}
+
+		for k, v := range deployTimes {
+			changeLog, ok := nginxChangeDiffs[k]
 			if !ok {
 				panic("not found " + k)
 			}