Merge pull request Kuadrant#555 from trepel/parametrize-no-of-scale-t…

…est-crs

Parametrize number of GWs and Listeners in scale test

scale_test/config.yaml

@@ -1,3 +1,5 @@
+{{- $KUADRANT_ZONE_ROOT_DOMAIN := .KUADRANT_ZONE_ROOT_DOMAIN }}
+{{- $NUM_LISTENERS := .NUM_LISTENERS }}
 ---
 metricsEndpoints: 
   - endpoint: {{ .PROMETHEUS_URL }}
@@ -50,50 +52,75 @@ jobs:
     namespace: scale-test
     waitWhenFinished: true
     objects:
+{{- $numGWs := .NUM_GWS | atoi }}
+{{- range $index := until $numGWs }}
+  {{- $GW_NUM := add1 $index }}
       - objectTemplate: ./gw.yaml
         replicas: 1
         waitOptions:
           forCondition: Programmed
           customStatusPath: ".conditions[].type"
         inputVars:
-          KUADRANT_ZONE_ROOT_DOMAIN: "{{ .KUADRANT_ZONE_ROOT_DOMAIN }}"
+          KUADRANT_ZONE_ROOT_DOMAIN: "{{$KUADRANT_ZONE_ROOT_DOMAIN}}"
+          NUM_LISTENERS: "{{$NUM_LISTENERS}}"
+          GW_NUM: "{{$GW_NUM}}"
       - objectTemplate: ./gw-tls-policy.yaml
         replicas: 1
         waitOptions:
           forCondition: Enforced
           customStatusPath: ".conditions[].type"
+        inputVars:
+          GW_NUM: "{{$GW_NUM}}"
       - objectTemplate: ./gw-dns-policy.yaml
         replicas: 1
         waitOptions:
           forCondition: Enforced
           customStatusPath: ".conditions[].type"
+        inputVars:
+          GW_NUM: "{{$GW_NUM}}"
       - objectTemplate: ./gw-rlp.yaml
         replicas: 1
         waitOptions:
           forCondition: Accepted
           customStatusPath: ".conditions[].type"
+        inputVars:
+          GW_NUM: "{{$GW_NUM}}"
       - objectTemplate: ./gw-auth-policy.yaml
         replicas: 1
         waitOptions:
           forCondition: Accepted
           customStatusPath: ".conditions[].type"
+        inputVars:
+          GW_NUM: "{{$GW_NUM}}"
+  {{- range $index := until ($NUM_LISTENERS | atoi)}}
+    {{- $LISTENER_NUM := add1 $index }}
       - objectTemplate: ./httproute.yaml
         replicas: 1
         waitOptions:
           forCondition: Accepted
           customStatusPath: ".conditions[].type"
         inputVars:
-          KUADRANT_ZONE_ROOT_DOMAIN: "{{ .KUADRANT_ZONE_ROOT_DOMAIN }}"
+          KUADRANT_ZONE_ROOT_DOMAIN: "{{$KUADRANT_ZONE_ROOT_DOMAIN}}"
+          GW_NUM: "{{$GW_NUM}}"
+          LISTENER_NUM: "{{$LISTENER_NUM}}"
       - objectTemplate: ./httproute-rlp.yaml
         replicas: 1
         waitOptions:
           forCondition: Enforced
           customStatusPath: ".conditions[].type"
+        inputVars:
+          GW_NUM: "{{$GW_NUM}}"
+          LISTENER_NUM: "{{$LISTENER_NUM}}"
       - objectTemplate: ./httproute-auth-policy.yaml
         replicas: 1
         waitOptions:
           forCondition: Enforced
           customStatusPath: ".conditions[].type"
+        inputVars:
+          GW_NUM: "{{$GW_NUM}}"
+          LISTENER_NUM: "{{$LISTENER_NUM}}"
+  {{- end }}
+{{- end }}
   - name: scale-test-safe-dnspolicy-cleanup
     jobType: delete
     jobIterations: 1

scale_test/gw-auth-policy.yaml

@@ -1,14 +1,15 @@
+{{- $GW_NUM := .GW_NUM }}
 apiVersion: kuadrant.io/v1beta2
 kind: AuthPolicy
 metadata:
-  name: gw-auth-policy-{{.Iteration}}
+  name: auth-policy-gw{{$GW_NUM}}-i{{ .Iteration }}
   labels: 
     app: scale-test
 spec:
   targetRef:
     group: gateway.networking.k8s.io
     kind: Gateway
-    name: scale-test-{{.Iteration}}
+    name: gw{{$GW_NUM}}-i{{ .Iteration }}
   rules:
     authorization:
       deny-all:

scale_test/gw-dns-policy.yaml

@@ -1,13 +1,14 @@
+{{- $GW_NUM := .GW_NUM }}
 apiVersion: kuadrant.io/v1alpha1
 kind: DNSPolicy
 metadata:
-  name: gw-dns-policy-{{.Iteration}}
+  name: dns-policy-gw{{$GW_NUM}}-i{{ .Iteration }}
   labels: 
     app: scale-test
 spec:
   targetRef:
     group: gateway.networking.k8s.io
     kind: Gateway
-    name: scale-test-{{.Iteration}}
+    name: gw{{$GW_NUM}}-i{{ .Iteration }}
   providerRefs:
   - name: aws-credentials

scale_test/gw-rlp.yaml

@@ -1,14 +1,15 @@
+{{- $GW_NUM := .GW_NUM }}
 apiVersion: kuadrant.io/v1beta3
 kind: RateLimitPolicy
 metadata:
-  name: gw-rlp-{{.Iteration}}
+  name: rlp-gw{{$GW_NUM}}-i{{ .Iteration }}
   labels: 
     app: scale-test
 spec:
   targetRef:
     group: gateway.networking.k8s.io
     kind: Gateway
-    name: scale-test-{{.Iteration}}
+    name: gw{{$GW_NUM}}-i{{ .Iteration }}
   limits:
     "global":
       rates:

scale_test/gw-tls-policy.yaml

@@ -1,14 +1,15 @@
+{{- $GW_NUM := .GW_NUM }}
 apiVersion: kuadrant.io/v1alpha1
 kind: TLSPolicy
 metadata:
-  name: gw-tls-policy-{{.Iteration}}
+  name: tls-policy-gw{{$GW_NUM}}-i{{ .Iteration }}
   labels: 
     app: scale-test
 spec:
   targetRef:
     group: gateway.networking.k8s.io
     kind: Gateway
-    name: scale-test-{{.Iteration}}
+    name: gw{{$GW_NUM}}-i{{ .Iteration }}
   issuerRef:
     group: cert-manager.io
     kind: ClusterIssuer

scale_test/gw.yaml

@@ -1,21 +1,28 @@
+{{- $Iteration := .Iteration }}
+{{- $KUADRANT_ZONE_ROOT_DOMAIN := .KUADRANT_ZONE_ROOT_DOMAIN }}
+{{- $GW_NUM := .GW_NUM }}
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata: 
-  name: scale-test-{{.Iteration}}
+  name: gw{{$GW_NUM}}-i{{$Iteration}}
   labels: 
     app: scale-test
 spec: 
   gatewayClassName: istio
   listeners: 
+{{- $numListeners := .NUM_LISTENERS | atoi }}
+{{- range $index := until $numListeners }}
+  {{- $LISTENER_NUM := add1 $index }}
   - allowedRoutes: 
       namespaces: 
         from: All
-    hostname: "*.scale-test-{{.Iteration}}.{{ .KUADRANT_ZONE_ROOT_DOMAIN }}"
-    name: api
+    hostname: "*.scale-test-gw{{$GW_NUM}}-l{{ $LISTENER_NUM }}-i{{$Iteration}}.{{ $KUADRANT_ZONE_ROOT_DOMAIN }}"
+    name: api-{{ $LISTENER_NUM }}
     port: 443
     protocol: HTTPS
     tls:
       mode: Terminate
       certificateRefs:
-        - name: scale-test-{{.Iteration}}
+        - name: cert-gw{{$GW_NUM}}-l{{ $LISTENER_NUM }}-i{{$Iteration}}
           kind: Secret
+{{- end }}

scale_test/httproute-auth-policy.yaml

@@ -1,14 +1,16 @@
+{{- $GW_NUM := .GW_NUM }}
+{{- $LISTENER_NUM := .LISTENER_NUM }}
 apiVersion: kuadrant.io/v1beta2
 kind: AuthPolicy
 metadata:
-  name: httproute-auth-policy-{{.Iteration}}
+  name: httproute-auth-policy-gw{{$GW_NUM}}-l{{$LISTENER_NUM}}-i{{ .Iteration }}
   labels: 
     app: scale-test
 spec:
   targetRef:
     group: gateway.networking.k8s.io
     kind: HTTPRoute
-    name: scale-test-{{.Iteration}}
+    name: httproute-gw{{$GW_NUM}}-l{{$LISTENER_NUM}}-i{{ .Iteration }}
   rules:
     authorization:
       allow-all:

scale_test/httproute-rlp.yaml

@@ -1,14 +1,16 @@
+{{- $GW_NUM := .GW_NUM }}
+{{- $LISTENER_NUM := .LISTENER_NUM }}
 apiVersion: kuadrant.io/v1beta3
 kind: RateLimitPolicy
 metadata:
-  name: httproute-rlp-{{.Iteration}}
+  name: httproute-rlp-gw{{$GW_NUM}}-l{{$LISTENER_NUM}}-i{{ .Iteration }}
   labels: 
     app: scale-test
 spec:
   targetRef:
     group: gateway.networking.k8s.io
     kind: HTTPRoute
-    name: scale-test-{{.Iteration}}
+    name: httproute-gw{{$GW_NUM}}-l{{$LISTENER_NUM}}-i{{ .Iteration }}
   limits:
     "httproute-level":
       rates:

scale_test/httproute.yaml

@@ -1,16 +1,18 @@
+{{- $GW_NUM := .GW_NUM }}
+{{- $LISTENER_NUM := .LISTENER_NUM }}
 apiVersion: gateway.networking.k8s.io/v1
 kind: HTTPRoute
 metadata: 
-  name: scale-test-{{.Iteration}}
+  name: httproute-gw{{$GW_NUM}}-l{{$LISTENER_NUM}}-i{{ .Iteration }}
   labels: 
     app: scale-test
 spec:
   parentRefs:
   - group: gateway.networking.k8s.io
     kind: Gateway
-    name: scale-test-{{.Iteration}}
+    name: gw{{$GW_NUM}}-i{{ .Iteration }}
   hostnames:
-  - "api-{{.Iteration}}.scale-test-{{.Iteration}}.{{ .KUADRANT_ZONE_ROOT_DOMAIN }}"
+  - "api.scale-test-gw{{$GW_NUM}}-l{{$LISTENER_NUM}}-i{{.Iteration}}.{{ .KUADRANT_ZONE_ROOT_DOMAIN }}"
   rules:
   - backendRefs:
     - group: ''

scale_test/readme.md

@@ -1,12 +1,12 @@
 # Control Plane Scale Test
 
-Control Plane scale testing via kube-burner utility
+Control Plane scale testing via kube-burner utility. It creates `NUM_GWS` Gateways each having `NUM_LISTENERS` listeners configured. For each Gateway one policy of each Kind (AuthPolicy, DNSPolicy, RateLimitPolicy, TLSPolicy) is created. For each listener one AuthPolicy and one RateLimitPolicy is created.
 
-## Prerequisities
+## Prerequisites
 
-This test assumes that Kuadrant together with all the dependencies (Gateway API, Istio, Certificate Manager etc) is installed. A ClusterIssuer (self-signed one is enough) is expected to exist too. Also make sure to port-forward Prometheus instance so that it is possible for kube-burner to query it.
+This test assumes that Kuadrant together with all the dependencies (Gateway API, Istio, Certificate Manager etc) are installed. A ClusterIssuer (self-signed one is enough) is expected to exist too. Also make sure to port-forward Prometheus instance so that it is possible for kube-burner to query it.
 
-The following env vars will need to be set to run the tests:
+The following environment variables will need to be set to run the tests:
 
 ```
 export KUADRANT_AWS_SECRET_ACCESS_KEY=[key]
@@ -17,15 +17,26 @@ export PROMETHEUS_URL=http://127.0.0.1:9090
 export PROMETHEUS_TOKEN=""
 export OS_INDEXING=true   # if sending metrics to opensearch/elasticsearch
 export ES_SERVER=https://[user]:[password]@[host]:[port]
+
+export NUM_GWS=1
+export NUM_LISTENERS=1
+```
+
+If you want to disable indexing you need to explicitly set related environment variables to an empty string:
+```
+export OS_INDEXING= # to disable indexing
+export ES_SERVER= # to disable indexing
 ```
 
 ## Execution
 
-`kube-burner init -c ./config.yaml --timeout 5m`
+`kube-burner init -c ./config.yaml --timeout 5m --uuid scale-test-$(openssl rand -hex 3)`
+
+Don't forget to increase the timeout if larger number of CRs are to be created.
 
 ## Setting up a local cluster for execution
 
-Follow the instructions in the Prerequisities section.
+Follow the instructions in the Prerequisites section.
 
 Clone the [kuadrant-operator](https://github.com/Kuadrant/kuadrant-operator) repo:
 
@@ -54,13 +65,13 @@ spec:
 EOF
 ```
 
-Port forward to prometheus:
+Port forward to Prometheus:
 
 ```bash
 kubectl -n monitoring port-forward svc/prometheus-k8s 9090:9090
 ```
 
-Run kube-burner:
+Run kube-burner (described in more detail above):
 
 ```bash
 kube-burner init -c ./config.yaml --timeout 5m