generated from kubernetes/kubernetes-template-project
-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #40 from AlexanderStocks/astocks/add-e2e-tests
test: add e2e provider tests
- Loading branch information
Showing
5 changed files
with
306 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| # Testing Secrets Store Sync Controller with e2e provider | ||
|
|
||
| This directory contains e2e test scripts for the Secrets Store Sync Controller. Before running the e2e tests, install | ||
| the Secrets Store Sync Controller with the `e2e provider` as outlined [here](../../README.md#getting-started). | ||
|
|
||
| ## Running the tests | ||
|
|
||
| 1. To run the tests, from the root directory run: | ||
|
|
||
| ```shell | ||
| make run-e2e-provider-tests | ||
| ``` | ||
|
|
||
| ## Testing | ||
|
|
||
| This doc lists the different Secret Sync scenarios tested as part of CI. | ||
|
|
||
| ## E2E tests | ||
|
|
||
| | Test Description | E2E | | ||
| |--------------------------------------------------------------------------------------------------------|-----| | ||
| | Check if `secretproviderclasses` CRD is established | ✔️ | | ||
| | Check if `secretsyncs` CRD is established | ✔️ | | ||
| | Test if RBAC roles and role bindings exist | ✔️ | | ||
| | Deploy `e2e-providerspc` SecretProviderClass CRD | ✔️ | | ||
| | Deploy `e2e-providerspc` SecretSync CRD | ✔️ | | ||
| | Deploy SecretProviderClass and SecretSync in different namespaces and check that no secret is created | ✔️ | | ||
| | Deploy SecretProviderClass and SecretSync, ensure secret is created, then delete SecretSync and verify | ✔️ | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| #!/usr/bin/env bats | ||
|
|
||
| load helpers | ||
|
|
||
| BATS_RESOURCE_MANIFESTS_DIR=hack/localsetup | ||
| BATS_RESOURCE_YAML_DIR=test/bats/tests/e2e_provider | ||
| WAIT_TIME=60 | ||
| SLEEP_TIME=1 | ||
|
|
||
| @test "secretproviderclasses crd is established" { | ||
| kubectl wait --for condition=established --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io | ||
|
|
||
| run kubectl get crd/secretproviderclasses.secrets-store.csi.x-k8s.io | ||
| assert_success | ||
| } | ||
|
|
||
| @test "secretsync crd is established" { | ||
| kubectl wait --for condition=established --timeout=60s crd/secretsyncs.secret-sync.x-k8s.io | ||
|
|
||
| run kubectl get crd/secretsyncs.secret-sync.x-k8s.io | ||
| assert_success | ||
| } | ||
|
|
||
| @test "Test rbac roles and role bindings exist" { | ||
| run kubectl get clusterrole/secrets-store-sync-controller-manager-role | ||
| assert_success | ||
|
|
||
| run kubectl get clusterrolebinding/secrets-store-sync-controller-manager-rolebinding | ||
| assert_success | ||
| } | ||
|
|
||
| @test "[v1alpha1] validate secret creation and deletion with SecretProviderClass and SecretSync" { | ||
| kubectl create namespace test-v1alpha1 --dry-run=client -o yaml | kubectl apply -f - | ||
|
|
||
| # Create the SPC | ||
| kubectl apply -n test-v1alpha1 -f $BATS_RESOURCE_MANIFESTS_DIR/e2e-providerspc.yaml | ||
|
|
||
| cmd="kubectl get secretproviderclasses.secrets-store.csi.x-k8s.io/e2e-providerspc -n test-v1alpha1 -o yaml | grep e2e-providerspc" | ||
| wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd" | ||
|
|
||
| # Create the SecretSync | ||
| kubectl apply -n test-v1alpha1 -f $BATS_RESOURCE_MANIFESTS_DIR/e2e-secret-sync.yaml | ||
|
|
||
| cmd="kubectl get secretsyncs.secret-sync.x-k8s.io/sse2esecret -n test-v1alpha1 -o yaml | grep sse2esecret" | ||
| wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd" | ||
|
|
||
| # Retrieve the secret | ||
| cmd="kubectl get secret sse2esecret -n test-v1alpha1 -o yaml | grep 'apiVersion: secret-sync.x-k8s.io/v1alpha1'" | ||
| wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd" | ||
|
|
||
| # Check the data in the secret | ||
| expected_data="secret" | ||
| secret_data=$(kubectl get secret sse2esecret -n test-v1alpha1 -o jsonpath='{.data.bar}' | base64 --decode) | ||
| [ "$secret_data" = "$expected_data" ] | ||
| # Check owner_count is 1 | ||
| cmd="compare_owner_count sse2esecret test-v1alpha1 1" | ||
| wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd" | ||
|
|
||
| # Delete the SecretSync | ||
| cmd="kubectl delete secretsync sse2esecret -n test-v1alpha1" | ||
| wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd" | ||
|
|
||
| # Check that the secret is deleted | ||
| cmd="kubectl get secret sse2esecret -n test-v1alpha1" | ||
| wait_for_process $WAIT_TIME $SLEEP_TIME "! $cmd" | ||
| } | ||
|
|
||
| @test "SecretProviderClass and SecretSync are deployed in different namespaces" { | ||
| # Create namespaces | ||
| kubectl create namespace spc-namespace --dry-run=client -o yaml | kubectl apply -f - | ||
| kubectl create namespace ss-namespace --dry-run=client -o yaml | kubectl apply -f - | ||
|
|
||
| # Deploy the SecretProviderClass in spc-namespace | ||
| kubectl apply -n spc-namespace -f $BATS_RESOURCE_MANIFESTS_DIR/e2e-providerspc.yaml | ||
|
|
||
| cmd="kubectl get secretproviderclasses.secrets-store.csi.x-k8s.io/e2e-providerspc -n spc-namespace -o yaml | grep e2e-providerspc" | ||
| wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd" | ||
|
|
||
| # Deploy the SecretSync in ss-namespace | ||
| kubectl apply -n ss-namespace -f $BATS_RESOURCE_MANIFESTS_DIR/e2e-secret-sync.yaml | ||
|
|
||
| cmd="kubectl get secretsyncs.secret-sync.x-k8s.io/sse2esecret -n ss-namespace -o yaml | grep sse2esecret" | ||
| wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd" | ||
|
|
||
| # Check the status of SecretSync in ss-namespace | ||
| status=$(kubectl get secretsyncs.secret-sync.x-k8s.io/sse2esecret -n ss-namespace -o jsonpath='{.status.conditions[0]}') | ||
|
|
||
| expected_message="Secret update failed because the controller could not retrieve the Secret Provider Class or the SPC is misconfigured. Check the logs or the events for more information." | ||
| expected_reason="ControllerSPCError" | ||
| expected_status="False" | ||
|
|
||
| # Extract individual fields from the status | ||
| message=$(echo $status | jq -r .message) | ||
| reason=$(echo $status | jq -r .reason) | ||
| status_value=$(echo $status | jq -r .status) | ||
|
|
||
| # Verify the status fields | ||
| [ "$message" = "$expected_message" ] | ||
| [ "$reason" = "$expected_reason" ] | ||
| [ "$status_value" = "$expected_status" ] | ||
|
|
||
| # Check that the secret is not created in ss-namespace | ||
| cmd="kubectl get secret sse2esecret -n ss-namespace" | ||
| run $cmd | ||
| assert_failure | ||
| } | ||
|
|
||
| teardown_file() { | ||
| archive_provider "app=secrets-store-sync-controller" || true | ||
| archive_info || true | ||
|
|
||
| if [[ "${INPLACE_UPGRADE_TEST}" != "true" ]]; then | ||
| #cleanup | ||
| run kubectl delete namespace test-v1alpha1 | ||
| run kubectl delete namespace spc-namespace | ||
| run kubectl delete namespace ss-namespace | ||
| echo "Done cleaning up e2e tests" | ||
| fi | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| #!/bin/bash | ||
|
|
||
| assert_success() { | ||
| if [[ "${status:-}" != 0 ]]; then | ||
| echo "expected: 0" | ||
| echo "actual: ${status:-}" | ||
| echo "output: ${output:-}" | ||
| return 1 | ||
| fi | ||
| } | ||
|
|
||
| assert_failure() { | ||
| if [[ "${status:-}" == 0 ]]; then | ||
| echo "expected: non-zero exit code" | ||
| echo "actual: ${status:-}" | ||
| echo "output: ${output:-}" | ||
| return 1 | ||
| fi | ||
| } | ||
|
|
||
| archive_provider() { | ||
| # Determine log directory | ||
| if [[ -z "${ARTIFACTS}" ]]; then | ||
| return 0 | ||
| fi | ||
|
|
||
| FILE_PREFIX=$(date +"%FT%H%M%S") | ||
|
|
||
| kubectl logs -l "$1" --tail -1 -n secrets-store-sync-controller-system > "${ARTIFACTS}/${FILE_PREFIX}-provider.logs" | ||
| } | ||
|
|
||
| archive_info() { | ||
| # Determine log directory | ||
| if [[ -z "${ARTIFACTS}" ]]; then | ||
| return 0 | ||
| fi | ||
|
|
||
| LOGS_DIR="${ARTIFACTS}/$(date +"%FT%H%M%S")" | ||
| mkdir -p "${LOGS_DIR}" | ||
|
|
||
| # print all pod information | ||
| kubectl get pods -A -o json > "${LOGS_DIR}/pods.json" | ||
|
|
||
| # print detailed pod information | ||
| kubectl describe pods --all-namespaces > "${LOGS_DIR}/pods-describe.txt" | ||
|
|
||
| # print logs from the secrets-store-sync-controller | ||
| # | ||
| # assumes secrets-store-sync-controller is installed with helm into the `secrets-store-sync-controller-system` namespace which | ||
| # sets the `app` selector to `secrets-store-sync-controller`. | ||
| # | ||
| # Note: the yaml deployment would require `app=secrets-store-sync-controller` | ||
| kubectl logs -l app=secrets-store-sync-controller --tail -1 -c manager -n secrets-store-sync-controller-system > "${LOGS_DIR}/secrets-store-sync-controller.log" | ||
| kubectl logs -l app=secrets-store-sync-controller --tail -1 -c provider-e2e-installer -n secrets-store-sync-controller-system > "${LOGS_DIR}/e2e-provider.log" | ||
|
|
||
| # print client and server version information | ||
| kubectl version > "${LOGS_DIR}/kubectl-version.txt" | ||
|
|
||
| # print generic cluster information | ||
| kubectl cluster-info dump > "${LOGS_DIR}/cluster-info.txt" | ||
|
|
||
| # collect metrics | ||
| local curl_pod_name | ||
| curl_pod_name="curl-$(openssl rand -hex 5)" | ||
| kubectl run "${curl_pod_name}" -n default --image=curlimages/curl:7.75.0 --labels="test=metrics_test" --overrides='{"spec": { "nodeSelector": {"kubernetes.io/os": "linux"}}}' -- tail -f /dev/null | ||
| kubectl wait --for=condition=Ready --timeout=60s -n default pod "${curl_pod_name}" | ||
|
|
||
| for pod_ip in $(kubectl get pod -n secrets-store-sync-controller-system -l app=secrets-store-sync-controller -o jsonpath="{.items[*].status.podIP}") | ||
| do | ||
| kubectl exec -n default "${curl_pod_name}" -- curl -s http://"${pod_ip}":8085/metrics > "${LOGS_DIR}/${pod_ip}.metrics" | ||
| done | ||
|
|
||
| kubectl delete pod -n default "${curl_pod_name}" | ||
| } | ||
|
|
||
| compare_owner_count() { | ||
| secret="$1" | ||
| namespace="$2" | ||
| ownercount="$3" | ||
|
|
||
| [[ "$(kubectl get secret "${secret}" -n "${namespace}" -o json | jq '.metadata.ownerReferences | length')" -eq $ownercount ]] | ||
| } | ||
|
|
||
| wait_for_process() { | ||
| wait_time="$1" | ||
| sleep_time="$2" | ||
| cmd="$3" | ||
| while [ "$wait_time" -gt 0 ]; do | ||
| if eval "$cmd"; then | ||
| return 0 | ||
| else | ||
| sleep "$sleep_time" | ||
| wait_time=$((wait_time-sleep_time)) | ||
| fi | ||
| done | ||
| return 1 | ||
| } |