Fix security context

.envrc

@@ -2,15 +2,3 @@ if ! has nix_direnv_version || ! nix_direnv_version 2.3.0; then
   source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/2.3.0/direnvrc" "sha256-Dmd+j63L84wuzgyjITIfSxSD57Tx7v51DMxVZOsiUD8="
 fi
 use flake . --impure
-
-# Vault
-export VAULT_ADDR=http://127.0.0.1:8200
-
-# Kubernetes
-export KUBECONFIG=$DEVENV_STATE/kube/config
-export KIND_CLUSTER_NAME=vault-secrets-webhook
-
-# Helm
-export HELM_CACHE_HOME="$DEVENV_STATE/helm/cache"
-export HELM_CONFIG_HOME="$DEVENV_STATE/helm/config"
-export HELM_DATA_HOME="$DEVENV_STATE/helm/data"

.gitignore

@@ -1,5 +1,6 @@
 /.devenv/
 /.direnv/
+/.garden/
 /.pre-commit-config.yaml
 /bin/
 /build/

README.md

@@ -34,6 +34,8 @@ Run the test suite:
 
 ```shell
 make test
+
+make container-image
 make test-e2e-local
 ```
 
@@ -65,6 +67,43 @@ make stop
 make down
 ```
 
+### Running e2e tests
+
+The project comes with an e2e test suite that is mostly self-contained,
+but at the very least, you need Docker installed.
+
+By default, the suite launches a [KinD](https://kind.sigs.k8s.io/) cluster, deploys all necessary components and runs the test suite.
+This is a good option if you want to run the test suite to make sure everything works. This is also how the CI runs the test suite
+(with a few minor differences).
+
+You can run the test suite by running the following commands:
+
+```shell
+make container-image
+make test-e2e-local
+```
+
+Another way to run the test suite is using an existing cluster.
+This may be a better option if you want to debug tests or figure out why something isn't working.
+
+Set up a Kubernetes cluster of your liking. For example, launch a KinD cluster:
+
+```shell
+kind create cluster
+```
+
+Deploy the necessary components (including the webhook itself):
+
+```shell
+garden deploy
+```
+
+Run the test suite:
+
+```shell
+make BOOTSTRAP=false test-e2e
+```
+
 ## License
 
 The project is licensed under the [Apache 2.0 License](LICENSE).

e2e/deploy/vault-secrets-webhook/values.yaml

@@ -1,3 +1,6 @@
+env:
+  VAULT_IMAGE: hashicorp/vault:1.14.1
+
 replicaCount: 1
 
 image:
@@ -8,5 +11,12 @@ configmapFailurePolicy: "Fail"
 podsFailurePolicy: "Fail"
 secretsFailurePolicy: "Fail"
 
-env:
-  VAULT_IMAGE: hashicorp/vault:1.14.1
+namespaceSelector:
+  matchExpressions:
+    # https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-metadata-name
+    - key: kubernetes.io/metadata.name
+      operator: NotIn
+      values:
+        - kube-system
+        - vault-operator
+        - vault-secrets-webhook

e2e/deploy/vault/vault.yaml

@@ -46,7 +46,9 @@ spec:
   config:
     storage:
       file:
-        path: "${ .Env.VAULT_STORAGE_FILE }" # An example how Vault config environment interpolation can be used
+        # Does not work with Garden
+        # path: "$${ .Env.VAULT_STORAGE_FILE }" # An example how Vault config environment interpolation can be used
+        path: /vault/file
     listener:
       tcp:
         address: "0.0.0.0:8200"

e2e/main_test.go

@@ -22,6 +22,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -58,8 +59,11 @@ func TestMain(m *testing.M) {
 	}
 	log.SetLogger(klog.NewKlogr())
 
+	bootstrap := strings.ToLower(os.Getenv("BOOTSTRAP")) != "false"
+	useRealCluster := !bootstrap || strings.ToLower(os.Getenv("USE_REAL_CLUSTER")) == "true"
+
 	// Set up cluster
-	if os.Getenv("USE_REAL_CLUSTER") == "true" {
+	if useRealCluster {
 		path := conf.ResolveKubeConfigFile()
 		cfg := envconf.NewWithKubeConfig(path)
 
@@ -90,23 +94,28 @@ func TestMain(m *testing.M) {
 		}
 	}
 
-	// Install vault-operator
-	testenv.Setup(installVaultOperator)
-	testenv.Finish(uninstallVaultOperator, envfuncs.DeleteNamespace("vault-operator"))
+	if bootstrap {
+		// Install vault-operator
+		testenv.Setup(installVaultOperator)
+		testenv.Finish(uninstallVaultOperator, envfuncs.DeleteNamespace("vault-operator"))
 
-	testenv.Setup(envfuncs.CreateNamespace("vault-secrets-webhook"), installVaultSecretsWebhook)
-	testenv.Finish(uninstallVaultSecretsWebhook, envfuncs.DeleteNamespace("vault-secrets-webhook"))
+		testenv.Setup(envfuncs.CreateNamespace("vault-secrets-webhook"), installVaultSecretsWebhook)
+		testenv.Finish(uninstallVaultSecretsWebhook, envfuncs.DeleteNamespace("vault-secrets-webhook"))
 
-	// Set up test namespace
-	// ns := envconf.RandomName("webhook-test", 16)
-	// testenv.Setup(envfuncs.CreateNamespace(ns))
-	// testenv.Finish(envfuncs.DeleteNamespace(ns))
+		// Set up test namespace
+		// ns := envconf.RandomName("webhook-test", 16)
+		// testenv.Setup(envfuncs.CreateNamespace(ns))
+		// testenv.Finish(envfuncs.DeleteNamespace(ns))
 
-	// Unsealing and Vault access only works in the default namespace at the moment
-	testenv.Setup(useNamespace("default"))
+		// Unsealing and Vault access only works in the default namespace at the moment
+		testenv.Setup(useNamespace("default"))
 
-	testenv.Setup(installVault, waitForVaultTLS)
-	testenv.Finish(uninstallVault)
+		testenv.Setup(installVault, waitForVaultTLS)
+		testenv.Finish(uninstallVault)
+	} else {
+		// Unsealing and Vault access only works in the default namespace at the moment
+		testenv.Setup(useNamespace("default"))
+	}
 
 	os.Exit(testenv.Run(m))
 }

e2e/test/deployment-init-seccontext.yaml

@@ -0,0 +1,34 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: test-deployment-init-seccontext
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: test-deployment-init-seccontext
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: test-deployment-init-seccontext
+      annotations:
+        vault.security.banzaicloud.io/vault-addr: "https://vault.default.svc.cluster.local:8200"
+        vault.security.banzaicloud.io/vault-role: "default"
+        vault.security.banzaicloud.io/vault-tls-secret: vault-tls
+        # vault.security.banzaicloud.io/vault-skip-verify: "true"
+        vault.security.banzaicloud.io/vault-path: "kubernetes"
+        vault.security.banzaicloud.io/run-as-non-root: "true"
+        vault.security.banzaicloud.io/run-as-user: "1000"
+        vault.security.banzaicloud.io/run-as-group: "1000"
+    spec:
+      containers:
+        - name: alpine
+          image: alpine
+          command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"]
+          env:
+          - name: AWS_SECRET_ACCESS_KEY
+            value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY
+          resources:
+            limits:
+              memory: "128Mi"
+              cpu: "100m"

e2e/webhook_test.go

@@ -136,6 +136,26 @@ func TestPodMutation(t *testing.T) {
 
 			return ctx
 		}).
+		Assess("security context defaults are correct", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			r := cfg.Client().Resources()
+
+			pods := &v1.PodList{}
+
+			err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment"))
+			require.NoError(t, err)
+
+			if pods == nil || len(pods.Items) == 0 {
+				t.Fatal("no pods found")
+			}
+
+			securityContext := pods.Items[0].Spec.InitContainers[0].SecurityContext
+
+			assert.Nil(t, securityContext.RunAsNonRoot)
+			assert.Nil(t, securityContext.RunAsUser)
+			assert.Nil(t, securityContext.RunAsGroup)
+
+			return ctx
+		}).
 		Feature()
 
 	deploymentSeccontext := applyResource(features.New("deployment-seccontext"), "deployment-seccontext.yaml").
@@ -197,7 +217,48 @@ func TestPodMutation(t *testing.T) {
 		}).
 		Feature()
 
-	testenv.Test(t, deployment, deploymentSeccontext, deploymentTemplating)
+	deploymentInitSeccontext := applyResource(features.New("deployment-init-seccontext"), "deployment-init-seccontext.yaml").
+		Assess("available", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			deployment := &appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{Name: "test-deployment-init-seccontext", Namespace: cfg.Namespace()},
+			}
+
+			// wait for the deployment to become available
+			err := wait.For(conditions.New(cfg.Client().Resources()).DeploymentConditionMatch(deployment, appsv1.DeploymentAvailable, v1.ConditionTrue), wait.WithTimeout(2*time.Minute))
+			require.NoError(t, err)
+
+			return ctx
+		}).
+		Assess("security context is correct", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			r := cfg.Client().Resources()
+
+			pods := &v1.PodList{}
+
+			err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-init-seccontext"))
+			require.NoError(t, err)
+
+			if pods == nil || len(pods.Items) == 0 {
+				t.Fatal("no pods found")
+			}
+
+			// wait for the container to become available
+			err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(2*time.Minute))
+			require.NoError(t, err)
+
+			securityContext := pods.Items[0].Spec.InitContainers[0].SecurityContext
+
+			require.NotNil(t, securityContext.RunAsNonRoot)
+			assert.Equal(t, true, *securityContext.RunAsNonRoot)
+			require.NotNil(t, securityContext.RunAsUser)
+			assert.Equal(t, int64(1000), *securityContext.RunAsUser)
+			require.NotNil(t, securityContext.RunAsGroup)
+			assert.Equal(t, int64(1000), *securityContext.RunAsGroup)
+
+			return ctx
+		}).
+		Feature()
+
+	testenv.Test(t, deployment, deploymentSeccontext, deploymentTemplating, deploymentInitSeccontext)
 }
 
 func applyResource(builder *features.FeatureBuilder, file string) *features.FeatureBuilder {