diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 83498f5f7..76d6fb9fc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,27 +51,3 @@ jobs: cache- - name: Test run: make test-prod - - test-load: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - name: Install Go - uses: buildjet/setup-go@v5 - with: - go-version: 1.22.x - - name: Setup kubernetes cluster - uses: container-tools/kind-action@v2 - - name: Checkout code - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 - - uses: buildjet/cache@v4 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - .bin - key: cache-${{ hashFiles('**/go.sum') }}-${{ hashFiles('.bin/*') }} - restore-keys: | - cache- - - name: Test - run: make test-load diff --git a/Makefile b/Makefile index ffaad90a6..99e1fd0a2 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,10 @@ test: manifests generate fmt vet envtest ## Run tests. test-prod: manifests generate fmt vet envtest ## Run tests. $(MAKE) gotest-prod -test-load: manifests generate fmt vet envtest ## Run tests. +test-load: envtest ## Run tests. + kubectl delete events --all -n testns + kubectl delete deployments --all -n testns + kubectl delete pods --all -n testns $(MAKE) gotest-load .PHONY: gotest @@ -77,7 +80,7 @@ gotest-prod: .PHONY: gotest-load gotest-load: make -C fixtures/load k6 - LOAD_TEST=1 KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./tests -coverprofile cover.out + LOAD_TEST=1 KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -v ./tests -coverprofile cover.out .PHONY: env env: envtest ## Run tests. diff --git a/cmd/operator.go b/cmd/operator.go index 8faeb2f02..ff7dfad2b 100644 --- a/cmd/operator.go +++ b/cmd/operator.go @@ -83,12 +83,12 @@ func run(ctx dutyContext.Context, args []string) error { utilruntime.Must(v1.AddToScheme(scheme)) registerJobs(ctx, args) - scrapers.StartEventListener(ctx) + scrapers.StartEventListener(dutyCtx) go serve(dutyCtx) go tableUpdatesHandler(dutyCtx) - return launchKopper(ctx) + return launchKopper(dutyCtx) } func launchKopper(ctx dutyContext.Context) error { diff --git a/fixtures/load/load.ts b/fixtures/load/load.ts index 9afdd253d..a0dc25408 100644 --- a/fixtures/load/load.ts +++ b/fixtures/load/load.ts @@ -131,6 +131,38 @@ const namespaceSpec = { } } +const deploymentSpec = { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + name: "nginx", + namespace: ns, + }, + spec: { + replicas: 3, + selector: { + matchLabels: { + app: "nginx" + } + }, + template: { + metadata: { + labels: { + app: "nginx" + } + }, + spec: { + containers: [ + { + name: "nginx", + image: "nginx:alpine", + } + ] + } + } + } +}; + let count = 10 export default function() { kubernetes = new Kubernetes(); @@ -171,7 +203,7 @@ export default function() { // Write this to file console.log(`Crashing pod: ${podName} at ${new Date().toLocaleString()}`); - file.appendString('log.txt', `${podName},${new Date().toISOString()}\n`) + file.appendString('log.txt', `${podName},crash,${new Date().toISOString()}\n`) try { let response = proxyGet(kubernetes.get("Pod", podName, ns), "panic", 9898) @@ -191,4 +223,14 @@ export default function() { pods.map(function(pod) { console.log(` ${pod.metadata.name} ${pod.status.phase}: restarts=${pod.status.containerStatuses[0].restartCount}`); }); + + // Create deployment to scale up and down + console.log(`Creating nginx deployment`); + kubernetes.apply(JSON.stringify(deploymentSpec)) + + k6.sleep(5); + const deployment1Replica = JSON.parse(JSON.stringify(deploymentSpec)); + deployment1Replica.spec.replicas = 1; + file.appendString('log.txt', `${deploymentSpec.metadata.name},scaledown,${new Date().toISOString()}\n`) + kubernetes.apply(JSON.stringify(deployment1Replica)) } diff --git a/tests/load_test.go b/tests/load_test.go index 22d0180c6..eb417f434 100644 --- a/tests/load_test.go +++ b/tests/load_test.go @@ -16,6 +16,8 @@ import ( "github.com/flanksource/duty/tests/setup" ginkgo "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" ) func TestLoad(t *testing.T) { @@ -27,10 +29,31 @@ var ( DefaultContext context.Context ) +type ChangeTimes []struct { + ChangeType string + CreatedAt time.Time + Name string + Details string +} + var _ = ginkgo.BeforeSuite(func() { + logger.Infof("Set defaukt context") DefaultContext = setup.BeforeSuiteFn() + kubeconfig := clientcmd.RecommendedHomeFile + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + panic(err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + panic(err) + } + DefaultContext = DefaultContext.WithKubernetes(clientset, config) + }) + var _ = ginkgo.AfterSuite(setup.AfterSuiteFn) var _ = ginkgo.Describe("Load Test", ginkgo.Ordered, func() { @@ -50,6 +73,7 @@ var _ = ginkgo.Describe("Load Test", ginkgo.Ordered, func() { Watch: []v1.KubernetesResourceToWatch{ {ApiVersion: "v1", Kind: "Namespace"}, {ApiVersion: "v1", Kind: "Pod"}, + //{ApiVersion: "apps/v1", Kind: "Deployment"}, }, }}, }, @@ -74,6 +98,8 @@ var _ = ginkgo.Describe("Load Test", ginkgo.Ordered, func() { os.Remove("log.txt") + time.Sleep(15 * time.Second) + logger.Infof("Exec k6") cmd := exec.Command("../fixtures/load/k6", "run", "../fixtures/load/load.ts", "--insecure-skip-tls-verify") err = cmd.Run() if err != nil { @@ -81,31 +107,27 @@ var _ = ginkgo.Describe("Load Test", ginkgo.Ordered, func() { panic(err) } - time.Sleep(3 * time.Minute) + logger.Infof("End k6") + time.Sleep(2 * time.Minute) var count int64 Expect(scraperCtx.DB().Table("config_changes").Count(&count).Error).To(BeNil()) Expect(count).ToNot(Equal(int64(0))) - var changes []struct { - ChangeType string - CreatedAt time.Time - Name string - } - + var podinfoChanges ChangeTimes err = scraperCtx.DB().Raw(` SELECT cc.change_type, cc.created_at, ci.name FROM config_changes cc INNER JOIN config_items ci ON cc.config_id = ci.id WHERE ci.name LIKE 'podinfo%' - `).Scan(&changes).Error + `).Scan(&podinfoChanges).Error Expect(err).To(BeNil()) - changeDiffs := make(map[string]time.Time) - for _, c := range changes { + podinfoChangeDiffs := make(map[string]time.Time) + for _, c := range podinfoChanges { logger.Infof("Change is %v", c) if c.ChangeType == v1.ChangeTypeDiff { - changeDiffs[c.Name] = c.CreatedAt + podinfoChangeDiffs[c.Name] = c.CreatedAt } } @@ -114,21 +136,56 @@ var _ = ginkgo.Describe("Load Test", ginkgo.Ordered, func() { lines := strings.Split(string(f), "\n") k6CrashTime := make(map[string]time.Time) + deployTimes := make(map[string]time.Time) for _, line := range lines { if strings.TrimSpace(line) == "" { continue } parts := strings.Split(line, ",") - t, err := time.Parse(time.RFC3339, parts[1]) + t, err := time.Parse(time.RFC3339, parts[2]) Expect(err).To(BeNil()) - k6CrashTime[parts[0]] = t + if parts[1] == "crash" { + k6CrashTime[parts[0]] = t + } else if parts[1] == "scaledown" { + deployTimes[parts[0]] = t + } logger.Infof("N=%s t=%s", parts[0], t) } for k, v := range k6CrashTime { - changeLog, ok := changeDiffs[k] + changeLog, ok := podinfoChangeDiffs[k] + if !ok { + panic("not found " + k) + } + td := changeLog.Sub(v) + logger.Infof("Delta for %s is %v", k, td) + Expect(td).To(BeNumerically("<", time.Minute)) + } + + var nginxChanges ChangeTimes + err = scraperCtx.DB().Raw(` + SELECT cc.change_type, cc.created_at, ci.name FROM config_changes cc + INNER JOIN config_items ci ON cc.config_id = ci.id + WHERE ci.name LIKE 'nginx%' + ORDER BY cc.created_at ASC + `).Scan(&nginxChanges).Error + + Expect(err).To(BeNil()) + + nginxChangeDiffs := make(map[string]time.Time) + nginxCounter := 0 + for _, c := range nginxChanges { + logger.Infof("Nginx change is %v", c) + if c.ChangeType == "ScalingReplicaSet" && nginxCounter != 0 { + nginxChangeDiffs[c.Name] = c.CreatedAt + } + nginxCounter += 1 + } + + for k, v := range deployTimes { + changeLog, ok := nginxChangeDiffs[k] if !ok { panic("not found " + k) }