From ca1339047c32a60940bf06d6d5865e8daf68463e Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Fri, 11 Aug 2023 15:48:58 +0200 Subject: [PATCH 1/4] feat: garden setup to make local testing easier Signed-off-by: Mark Sagi-Kazar --- .envrc | 12 - .gitignore | 1 + e2e/deploy/vault-secrets-webhook/values.yaml | 14 +- e2e/deploy/vault/vault.yaml | 4 +- e2e/main_test.go | 37 ++- flake.lock | 311 ++++++++++++++++++- flake.nix | 32 +- garden.yaml | 50 +++ project.garden.yaml | 23 ++ 9 files changed, 426 insertions(+), 58 deletions(-) create mode 100644 garden.yaml create mode 100644 project.garden.yaml diff --git a/.envrc b/.envrc index 694a717f..3ce7171a 100644 --- a/.envrc +++ b/.envrc @@ -2,15 +2,3 @@ if ! has nix_direnv_version || ! nix_direnv_version 2.3.0; then source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/2.3.0/direnvrc" "sha256-Dmd+j63L84wuzgyjITIfSxSD57Tx7v51DMxVZOsiUD8=" fi use flake . --impure - -# Vault -export VAULT_ADDR=http://127.0.0.1:8200 - -# Kubernetes -export KUBECONFIG=$DEVENV_STATE/kube/config -export KIND_CLUSTER_NAME=vault-secrets-webhook - -# Helm -export HELM_CACHE_HOME="$DEVENV_STATE/helm/cache" -export HELM_CONFIG_HOME="$DEVENV_STATE/helm/config" -export HELM_DATA_HOME="$DEVENV_STATE/helm/data" diff --git a/.gitignore b/.gitignore index 5a3cd4b3..155edd85 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /.devenv/ /.direnv/ +/.garden/ /.pre-commit-config.yaml /bin/ /build/ diff --git a/e2e/deploy/vault-secrets-webhook/values.yaml b/e2e/deploy/vault-secrets-webhook/values.yaml index 5a8909c1..a42bbdc9 100644 --- a/e2e/deploy/vault-secrets-webhook/values.yaml +++ b/e2e/deploy/vault-secrets-webhook/values.yaml @@ -1,3 +1,6 @@ +env: + VAULT_IMAGE: hashicorp/vault:1.14.1 + replicaCount: 1 image: @@ -8,5 +11,12 @@ configmapFailurePolicy: "Fail" podsFailurePolicy: "Fail" secretsFailurePolicy: "Fail" -env: - VAULT_IMAGE: hashicorp/vault:1.14.1 +namespaceSelector: + matchExpressions: + # https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-metadata-name + - key: kubernetes.io/metadata.name + operator: NotIn + values: + - kube-system + - vault-operator + - vault-secrets-webhook diff --git a/e2e/deploy/vault/vault.yaml b/e2e/deploy/vault/vault.yaml index 4c728769..aa81b460 100644 --- a/e2e/deploy/vault/vault.yaml +++ b/e2e/deploy/vault/vault.yaml @@ -46,7 +46,9 @@ spec: config: storage: file: - path: "${ .Env.VAULT_STORAGE_FILE }" # An example how Vault config environment interpolation can be used + # Does not work with Garden + # path: "$${ .Env.VAULT_STORAGE_FILE }" # An example how Vault config environment interpolation can be used + path: /vault/file listener: tcp: address: "0.0.0.0:8200" diff --git a/e2e/main_test.go b/e2e/main_test.go index a5460b9a..092e5bf9 100644 --- a/e2e/main_test.go +++ b/e2e/main_test.go @@ -22,6 +22,7 @@ import ( "flag" "fmt" "os" + "strings" "testing" "time" @@ -58,8 +59,11 @@ func TestMain(m *testing.M) { } log.SetLogger(klog.NewKlogr()) + bootstrap := strings.ToLower(os.Getenv("BOOTSTRAP")) != "false" + useRealCluster := !bootstrap || strings.ToLower(os.Getenv("USE_REAL_CLUSTER")) == "true" + // Set up cluster - if os.Getenv("USE_REAL_CLUSTER") == "true" { + if useRealCluster { path := conf.ResolveKubeConfigFile() cfg := envconf.NewWithKubeConfig(path) @@ -90,23 +94,28 @@ func TestMain(m *testing.M) { } } - // Install vault-operator - testenv.Setup(installVaultOperator) - testenv.Finish(uninstallVaultOperator, envfuncs.DeleteNamespace("vault-operator")) + if bootstrap { + // Install vault-operator + testenv.Setup(installVaultOperator) + testenv.Finish(uninstallVaultOperator, envfuncs.DeleteNamespace("vault-operator")) - testenv.Setup(envfuncs.CreateNamespace("vault-secrets-webhook"), installVaultSecretsWebhook) - testenv.Finish(uninstallVaultSecretsWebhook, envfuncs.DeleteNamespace("vault-secrets-webhook")) + testenv.Setup(envfuncs.CreateNamespace("vault-secrets-webhook"), installVaultSecretsWebhook) + testenv.Finish(uninstallVaultSecretsWebhook, envfuncs.DeleteNamespace("vault-secrets-webhook")) - // Set up test namespace - // ns := envconf.RandomName("webhook-test", 16) - // testenv.Setup(envfuncs.CreateNamespace(ns)) - // testenv.Finish(envfuncs.DeleteNamespace(ns)) + // Set up test namespace + // ns := envconf.RandomName("webhook-test", 16) + // testenv.Setup(envfuncs.CreateNamespace(ns)) + // testenv.Finish(envfuncs.DeleteNamespace(ns)) - // Unsealing and Vault access only works in the default namespace at the moment - testenv.Setup(useNamespace("default")) + // Unsealing and Vault access only works in the default namespace at the moment + testenv.Setup(useNamespace("default")) - testenv.Setup(installVault, waitForVaultTLS) - testenv.Finish(uninstallVault) + testenv.Setup(installVault, waitForVaultTLS) + testenv.Finish(uninstallVault) + } else { + // Unsealing and Vault access only works in the default namespace at the moment + testenv.Setup(useNamespace("default")) + } os.Exit(testenv.Run(m)) } diff --git a/flake.lock b/flake.lock index 8becf620..56e250a6 100644 --- a/flake.lock +++ b/flake.lock @@ -8,11 +8,32 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1689504341, - "narHash": "sha256-btK/nUaxB6HBCiuhnebiCTnohatHElVwWrBOWP+oWpI=", + "lastModified": 1691592502, + "narHash": "sha256-Ur44RgRVhYRDbTYHmzBquRC5KcvEuk0zKd6UJ2nHHRU=", "owner": "cachix", "repo": "devenv", - "rev": "b9c9d83e89c0405ab02880132a0601911c477250", + "rev": "507bdcad35253545d43a1fc0898da6ecebb6b52a", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "devenv_2": { + "inputs": { + "flake-compat": "flake-compat_2", + "nix": "nix_2", + "nixpkgs": "nixpkgs_2", + "pre-commit-hooks": "pre-commit-hooks_2" + }, + "locked": { + "lastModified": 1691592502, + "narHash": "sha256-Ur44RgRVhYRDbTYHmzBquRC5KcvEuk0zKd6UJ2nHHRU=", + "owner": "cachix", + "repo": "devenv", + "rev": "507bdcad35253545d43a1fc0898da6ecebb6b52a", "type": "github" }, "original": { @@ -37,16 +58,50 @@ "type": "github" } }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-parts": { "inputs": { "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1688466019, - "narHash": "sha256-VeM2akYrBYMsb4W/MmBo1zmaMfgbL4cH3Pu8PGyIwJ0=", + "lastModified": 1690933134, + "narHash": "sha256-ab989mN63fQZBFrkk4Q8bYxQCktuHmBIBqUG1jl6/FQ=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "59cf3f1447cfc75087e7273b04b31e689a8599fb", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_2": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib_2" + }, + "locked": { + "lastModified": 1690933134, + "narHash": "sha256-ab989mN63fQZBFrkk4Q8bYxQCktuHmBIBqUG1jl6/FQ=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "8e8d955c22df93dbe24f19ea04f47a74adbdc5ec", + "rev": "59cf3f1447cfc75087e7273b04b31e689a8599fb", "type": "github" }, "original": { @@ -73,6 +128,44 @@ "type": "github" } }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1685518550, + "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "garden": { + "inputs": { + "devenv": "devenv_2", + "flake-parts": "flake-parts_2", + "nixpkgs": "nixpkgs_3" + }, + "locked": { + "lastModified": 1692272136, + "narHash": "sha256-KpDwEWZB61zC6MM5jjfS2QSMOBszG1W96Llcgs1NRMc=", + "owner": "sagikazarmark", + "repo": "nix-garden", + "rev": "8d05717d369ed55a06bc294e51142fd477d23508", + "type": "github" + }, + "original": { + "owner": "sagikazarmark", + "repo": "nix-garden", + "type": "github" + } + }, "gitignore": { "inputs": { "nixpkgs": [ @@ -95,6 +188,29 @@ "type": "github" } }, + "gitignore_2": { + "inputs": { + "nixpkgs": [ + "garden", + "devenv", + "pre-commit-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1660459072, + "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, "lowdown-src": { "flake": false, "locked": { @@ -111,6 +227,22 @@ "type": "github" } }, + "lowdown-src_2": { + "flake": false, + "locked": { + "lastModified": 1633514407, + "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", + "owner": "kristapsdz", + "repo": "lowdown", + "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", + "type": "github" + }, + "original": { + "owner": "kristapsdz", + "repo": "lowdown", + "type": "github" + } + }, "nix": { "inputs": { "lowdown-src": "lowdown-src", @@ -135,6 +267,31 @@ "type": "github" } }, + "nix_2": { + "inputs": { + "lowdown-src": "lowdown-src_2", + "nixpkgs": [ + "garden", + "devenv", + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression_2" + }, + "locked": { + "lastModified": 1676545802, + "narHash": "sha256-EK4rZ+Hd5hsvXnzSzk2ikhStJnD63odF7SzsQ8CuSPU=", + "owner": "domenkozar", + "repo": "nix", + "rev": "7c91803598ffbcfe4a55c44ac6d49b2cf07a527f", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "relaxed-flakes", + "repo": "nix", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1678875422, @@ -154,11 +311,29 @@ "nixpkgs-lib": { "locked": { "dir": "lib", - "lastModified": 1688049487, - "narHash": "sha256-100g4iaKC9MalDjUW9iN6Jl/OocTDtXdeAj7pEGIRh4=", + "lastModified": 1690881714, + "narHash": "sha256-h/nXluEqdiQHs1oSgkOOWF+j8gcJMWhwnZ9PFabN6q0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "4bc72cae107788bf3f24f30db2e2f685c9298dc9", + "rev": "9e1960bc196baf6881340d53dccb203a951745a2", + "type": "github" + }, + "original": { + "dir": "lib", + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib_2": { + "locked": { + "dir": "lib", + "lastModified": 1690881714, + "narHash": "sha256-h/nXluEqdiQHs1oSgkOOWF+j8gcJMWhwnZ9PFabN6q0=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9e1960bc196baf6881340d53dccb203a951745a2", "type": "github" }, "original": { @@ -185,6 +360,22 @@ "type": "github" } }, + "nixpkgs-regression_2": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, "nixpkgs-stable": { "locked": { "lastModified": 1685801374, @@ -201,13 +392,61 @@ "type": "github" } }, + "nixpkgs-stable_2": { + "locked": { + "lastModified": 1685801374, + "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c37ca420157f4abc31e26f436c1145f8951ff373", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.05", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs_2": { "locked": { - "lastModified": 1689510077, - "narHash": "sha256-sFOtguHP6enqXsySLREIrevxwz/P9Da/9XQwiIiTK38=", + "lastModified": 1678875422, + "narHash": "sha256-T3o6NcQPwXjxJMn2shz86Chch4ljXgZn746c2caGxd8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "126f49a01de5b7e35a43fd43f891ecf6d3a51459", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1691625043, + "narHash": "sha256-IiiOwgRTQm9W1QHe8qme7qYxDbAT2MYxbIJMfPEltN0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "12d2f0da28a47f907825939630b4a0e8122fabe3", + "rev": "3d6ebeb283be256f008541ce2b089eb5fb0e4e01", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1691683125, + "narHash": "sha256-FMU62G57HDbJwU+9V3q7I0mBaQYTYQdtPNlJt2t5/A4=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "4d2389b927696ef8da4ef76b03f2d306faf87929", "type": "github" }, "original": { @@ -245,11 +484,42 @@ "type": "github" } }, + "pre-commit-hooks_2": { + "inputs": { + "flake-compat": [ + "garden", + "devenv", + "flake-compat" + ], + "flake-utils": "flake-utils_2", + "gitignore": "gitignore_2", + "nixpkgs": [ + "garden", + "devenv", + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable_2" + }, + "locked": { + "lastModified": 1688056373, + "narHash": "sha256-2+SDlNRTKsgo3LBRiMUcoEUb6sDViRNQhzJquZ4koOI=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "5843cf069272d92b60c3ed9e55b7a8989c01d4c7", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, "root": { "inputs": { "devenv": "devenv", "flake-parts": "flake-parts", - "nixpkgs": "nixpkgs_2" + "garden": "garden", + "nixpkgs": "nixpkgs_4" } }, "systems": { @@ -266,6 +536,21 @@ "repo": "default", "type": "github" } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 10a9e0f6..edade3d9 100644 --- a/flake.nix +++ b/flake.nix @@ -5,6 +5,7 @@ nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; flake-parts.url = "github:hercules-ci/flake-parts"; devenv.url = "github:cachix/devenv"; + garden.url = "github:sagikazarmark/nix-garden"; }; outputs = inputs@{ flake-parts, ... }: @@ -43,30 +44,29 @@ kubernetes-helm helm-docs + k3d + + crc + yamllint hadolint ] ++ [ + inputs'.garden.packages.garden self'.packages.licensei ]; - scripts = { - versions.exec = '' - go version - golangci-lint version - kind version - kubectl version --client - echo kustomize $(kustomize version --short) - echo helm $(helm version --short) - ''; - }; + env = { + GARDEN_DISABLE_ANALYTICS = "true"; - enterShell = '' - # Vault - export VAULT_ADDR=http://127.0.0.1:8200 - export VAULT_TOKEN=227e1cce-6bf7-30bb-2d2a-acc854318caf + KUBECONFIG = "${config.devenv.shells.default.env.DEVENV_STATE}/kube/config"; + KIND_CLUSTER_NAME = "vault-secrets-webhook"; - versions - ''; + HELM_CACHE_HOME = "${config.devenv.shells.default.env.DEVENV_STATE}/helm/cache"; + HELM_CONFIG_HOME = "${config.devenv.shells.default.env.DEVENV_STATE}/helm/config"; + HELM_DATA_HOME = "${config.devenv.shells.default.env.DEVENV_STATE}/helm/data"; + + VAULT_TOKEN = "227e1cce-6bf7-30bb-2d2a-acc854318caf"; + }; # https://github.com/cachix/devenv/issues/528#issuecomment-1556108767 containers = pkgs.lib.mkForce { }; diff --git a/garden.yaml b/garden.yaml new file mode 100644 index 00000000..0f73ca78 --- /dev/null +++ b/garden.yaml @@ -0,0 +1,50 @@ +kind: Deploy +type: helm +name: vault-operator +spec: + namespace: vault-operator + chart: + name: oci://ghcr.io/bank-vaults/helm-charts/vault-operator + version: "1.20.0" + +--- +kind: Deploy +type: kubernetes +name: vault +dependencies: + - deploy.vault-operator +spec: + namespace: default + files: + - ./e2e/deploy/vault/rbac.yaml + - ./e2e/deploy/vault/vault.yaml + +--- +kind: Build +type: container +name: vault-secrets-webhook +exclude: + - .direnv/**/* + - .devenv/**/* + - build/**/* + - e2e/**/* + +--- +kind: Deploy +type: helm +name: vault-secrets-webhook +dependencies: + - deploy.vault +variables: + repository: ${actions.build.vault-secrets-webhook.outputs.deployment-image-name} + tag: ${actions.build.vault-secrets-webhook.version} +spec: + namespace: vault-secrets-webhook + chart: + path: ./deploy/charts/vault-secrets-webhook/ + valueFiles: + - ./e2e/deploy/vault-secrets-webhook/values.yaml + values: + image: + repository: ${var.repository} + tag: ${var.tag} diff --git a/project.garden.yaml b/project.garden.yaml new file mode 100644 index 00000000..5e18a9b7 --- /dev/null +++ b/project.garden.yaml @@ -0,0 +1,23 @@ +# Documentation about Garden projects can be found at https://docs.garden.io/using-garden/projects +# Reference for Garden projects can be found at https://docs.garden.io/reference/project-config +apiVersion: garden.io/v1 +kind: Project +name: vault-secrets-webhook +dotIgnoreFile: .gitignore +defaultEnvironment: local + +environments: + - name: local + defaultNamespace: default + +providers: + - name: local-kubernetes + environments: [local] + setupIngressController: null + +scan: + exclude: + - .direnv/**/* + - .devenv/**/* + - build/**/* + - e2e/**/* From a56aeac44d884b56c66d152a44c89f0498ca98d1 Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Thu, 31 Aug 2023 14:23:33 +0200 Subject: [PATCH 2/4] docs: document running e2e tests Signed-off-by: Mark Sagi-Kazar --- README.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/README.md b/README.md index 8209cb39..1534ebdc 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ Run the test suite: ```shell make test + +make container-image make test-e2e-local ``` @@ -65,6 +67,43 @@ make stop make down ``` +### Running e2e tests + +The project comes with an e2e test suite that is mostly self-contained, +but at the very least, you need Docker installed. + +By default, the suite launches a [KinD](https://kind.sigs.k8s.io/) cluster, deploys all necessary components and runs the test suite. +This is a good option if you want to run the test suite to make sure everything works. This is also how the CI runs the test suite +(with a few minor differences). + +You can run the test suite by running the following commands: + +```shell +make container-image +make test-e2e-local +``` + +Another way to run the test suite is using an existing cluster. +This may be a better option if you want to debug tests or figure out why something isn't working. + +Set up a Kubernetes cluster of your liking. For example, launch a KinD cluster: + +```shell +kind create cluster +``` + +Deploy the necessary components (including the webhook itself): + +```shell +garden deploy +``` + +Run the test suite: + +```shell +make BOOTSTRAP=false test-e2e +``` + ## License The project is licensed under the [Apache 2.0 License](LICENSE). From c5110520c26dbd99b925f3de8537abc49ea8fa82 Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Thu, 31 Aug 2023 14:24:03 +0200 Subject: [PATCH 3/4] fix: only set security context when necessary Signed-off-by: Mark Sagi-Kazar --- e2e/test/deployment-init-seccontext.yaml | 34 +++++++++++++ e2e/webhook_test.go | 63 +++++++++++++++++++++++- pkg/webhook/pod.go | 8 ++- pkg/webhook/pod_test.go | 21 ++++++++ 4 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 e2e/test/deployment-init-seccontext.yaml diff --git a/e2e/test/deployment-init-seccontext.yaml b/e2e/test/deployment-init-seccontext.yaml new file mode 100644 index 00000000..3c7042bc --- /dev/null +++ b/e2e/test/deployment-init-seccontext.yaml @@ -0,0 +1,34 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-init-seccontext +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: test-deployment-init-seccontext + template: + metadata: + labels: + app.kubernetes.io/name: test-deployment-init-seccontext + annotations: + vault.security.banzaicloud.io/vault-addr: "https://vault.default.svc.cluster.local:8200" + vault.security.banzaicloud.io/vault-role: "default" + vault.security.banzaicloud.io/vault-tls-secret: vault-tls + # vault.security.banzaicloud.io/vault-skip-verify: "true" + vault.security.banzaicloud.io/vault-path: "kubernetes" + vault.security.banzaicloud.io/run-as-non-root: "true" + vault.security.banzaicloud.io/run-as-user: "1000" + vault.security.banzaicloud.io/run-as-group: "1000" + spec: + containers: + - name: alpine + image: alpine + command: ["sh", "-c", "echo $AWS_SECRET_ACCESS_KEY && echo going to sleep... && sleep 10000"] + env: + - name: AWS_SECRET_ACCESS_KEY + value: vault:secret/data/accounts/aws#AWS_SECRET_ACCESS_KEY + resources: + limits: + memory: "128Mi" + cpu: "100m" diff --git a/e2e/webhook_test.go b/e2e/webhook_test.go index b63587ed..9e85f1a1 100644 --- a/e2e/webhook_test.go +++ b/e2e/webhook_test.go @@ -136,6 +136,26 @@ func TestPodMutation(t *testing.T) { return ctx }). + Assess("security context defaults are correct", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + r := cfg.Client().Resources() + + pods := &v1.PodList{} + + err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment")) + require.NoError(t, err) + + if pods == nil || len(pods.Items) == 0 { + t.Fatal("no pods found") + } + + securityContext := pods.Items[0].Spec.InitContainers[0].SecurityContext + + assert.Nil(t, securityContext.RunAsNonRoot) + assert.Nil(t, securityContext.RunAsUser) + assert.Nil(t, securityContext.RunAsGroup) + + return ctx + }). Feature() deploymentSeccontext := applyResource(features.New("deployment-seccontext"), "deployment-seccontext.yaml"). @@ -197,7 +217,48 @@ func TestPodMutation(t *testing.T) { }). Feature() - testenv.Test(t, deployment, deploymentSeccontext, deploymentTemplating) + deploymentInitSeccontext := applyResource(features.New("deployment-init-seccontext"), "deployment-init-seccontext.yaml"). + Assess("available", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "test-deployment-init-seccontext", Namespace: cfg.Namespace()}, + } + + // wait for the deployment to become available + err := wait.For(conditions.New(cfg.Client().Resources()).DeploymentConditionMatch(deployment, appsv1.DeploymentAvailable, v1.ConditionTrue), wait.WithTimeout(2*time.Minute)) + require.NoError(t, err) + + return ctx + }). + Assess("security context is correct", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + r := cfg.Client().Resources() + + pods := &v1.PodList{} + + err := r.List(ctx, pods, resources.WithLabelSelector("app.kubernetes.io/name=test-deployment-init-seccontext")) + require.NoError(t, err) + + if pods == nil || len(pods.Items) == 0 { + t.Fatal("no pods found") + } + + // wait for the container to become available + err = wait.For(conditions.New(r).ContainersReady(&pods.Items[0]), wait.WithTimeout(2*time.Minute)) + require.NoError(t, err) + + securityContext := pods.Items[0].Spec.InitContainers[0].SecurityContext + + require.NotNil(t, securityContext.RunAsNonRoot) + assert.Equal(t, true, *securityContext.RunAsNonRoot) + require.NotNil(t, securityContext.RunAsUser) + assert.Equal(t, int64(1000), *securityContext.RunAsUser) + require.NotNil(t, securityContext.RunAsGroup) + assert.Equal(t, int64(1000), *securityContext.RunAsGroup) + + return ctx + }). + Feature() + + testenv.Test(t, deployment, deploymentSeccontext, deploymentTemplating, deploymentInitSeccontext) } func applyResource(builder *features.FeatureBuilder, file string) *features.FeatureBuilder { diff --git a/pkg/webhook/pod.go b/pkg/webhook/pod.go index 2770c5f9..6573518c 100644 --- a/pkg/webhook/pod.go +++ b/pkg/webhook/pod.go @@ -845,8 +845,6 @@ func getAgentContainers(originalContainers []corev1.Container, podSecurityContex func getBaseSecurityContext(podSecurityContext *corev1.PodSecurityContext, vaultConfig VaultConfig) *corev1.SecurityContext { context := &corev1.SecurityContext{ AllowPrivilegeEscalation: &vaultConfig.PspAllowPrivilegeEscalation, - RunAsNonRoot: &vaultConfig.RunAsNonRoot, - RunAsUser: &vaultConfig.RunAsUser, ReadOnlyRootFilesystem: &vaultConfig.ReadOnlyRootFilesystem, Capabilities: &corev1.Capabilities{ Add: []corev1.Capability{ @@ -866,6 +864,12 @@ func getBaseSecurityContext(podSecurityContext *corev1.PodSecurityContext, vault context.RunAsUser = podSecurityContext.RunAsUser } + // Although it could explicitly be set to false, + // the behavior of false and unset are the same + if vaultConfig.RunAsNonRoot { + context.RunAsNonRoot = &vaultConfig.RunAsNonRoot + } + if vaultConfig.RunAsUser > 0 { context.RunAsUser = &vaultConfig.RunAsUser } diff --git a/pkg/webhook/pod_test.go b/pkg/webhook/pod_test.go index 5b4f3165..2bd05c6b 100644 --- a/pkg/webhook/pod_test.go +++ b/pkg/webhook/pod_test.go @@ -38,6 +38,9 @@ var vaultConfig = VaultConfig{ VaultEnvPassThrough: "vaultEnvPassThrough", EnableJSONLog: "enableJSONLog", ClientTimeout: 10 * time.Second, + RunAsNonRoot: true, + RunAsUser: int64(1000), + RunAsGroup: int64(1000), } type MockRegistry struct { @@ -506,6 +509,7 @@ func Test_mutatingWebhook_mutatePod(t *testing.T) { baseSecurityContext := &corev1.SecurityContext{ RunAsUser: &vaultConfig.RunAsUser, + RunAsGroup: &vaultConfig.RunAsGroup, RunAsNonRoot: &vaultConfig.RunAsNonRoot, ReadOnlyRootFilesystem: &vaultConfig.ReadOnlyRootFilesystem, AllowPrivilegeEscalation: &vaultConfig.PspAllowPrivilegeEscalation, @@ -525,6 +529,7 @@ func Test_mutatingWebhook_mutatePod(t *testing.T) { agentInitContainerSecurityContext := &corev1.SecurityContext{ RunAsUser: &vaultConfig.RunAsUser, + RunAsGroup: &vaultConfig.RunAsGroup, RunAsNonRoot: &vaultConfig.RunAsNonRoot, ReadOnlyRootFilesystem: &vaultConfig.ReadOnlyRootFilesystem, AllowPrivilegeEscalation: &vaultConfig.PspAllowPrivilegeEscalation, @@ -544,6 +549,7 @@ func Test_mutatingWebhook_mutatePod(t *testing.T) { agentContainerSecurityContext := &corev1.SecurityContext{ RunAsUser: &vaultConfig.RunAsUser, + RunAsGroup: &vaultConfig.RunAsGroup, RunAsNonRoot: &vaultConfig.RunAsNonRoot, ReadOnlyRootFilesystem: &vaultConfig.ReadOnlyRootFilesystem, AllowPrivilegeEscalation: &vaultConfig.PspAllowPrivilegeEscalation, @@ -609,6 +615,9 @@ func Test_mutatingWebhook_mutatePod(t *testing.T) { EnvCPULimit: resource.MustParse("250m"), EnvMemoryLimit: resource.MustParse("64Mi"), ServiceAccountTokenVolumeName: "/var/run/secrets/kubernetes.io/serviceaccount", + RunAsNonRoot: true, + RunAsUser: int64(1000), + RunAsGroup: int64(1000), }, }, wantedPod: &corev1.Pod{ @@ -803,6 +812,9 @@ func Test_mutatingWebhook_mutatePod(t *testing.T) { EnvCPULimit: resource.MustParse("250m"), EnvMemoryLimit: resource.MustParse("64Mi"), ServiceAccountTokenVolumeName: "/var/run/secrets/kubernetes.io/serviceaccount", + RunAsNonRoot: true, + RunAsUser: int64(1000), + RunAsGroup: int64(1000), }, }, wantedPod: &corev1.Pod{ @@ -993,6 +1005,9 @@ func Test_mutatingWebhook_mutatePod(t *testing.T) { AgentImagePullPolicy: "IfNotPresent", ServiceAccountTokenVolumeName: "/var/run/secrets/kubernetes.io/serviceaccount", AgentEnvVariables: "[{\"Name\": \"SKIP_SETCAP\",\"Value\": \"1\"}]", + RunAsNonRoot: true, + RunAsUser: int64(1000), + RunAsGroup: int64(1000), }, }, wantedPod: &corev1.Pod{ @@ -1155,6 +1170,9 @@ func Test_mutatingWebhook_mutatePod(t *testing.T) { EnvCPULimit: resource.MustParse("250m"), EnvMemoryLimit: resource.MustParse("64Mi"), ServiceAccountTokenVolumeName: "/var/run/secrets/kubernetes.io/serviceaccount", + RunAsNonRoot: true, + RunAsUser: int64(1000), + RunAsGroup: int64(1000), }, }, wantedPod: &corev1.Pod{ @@ -1374,6 +1392,9 @@ func Test_mutatingWebhook_mutatePod(t *testing.T) { EnvCPULimit: resource.MustParse("250m"), EnvMemoryLimit: resource.MustParse("64Mi"), ServiceAccountTokenVolumeName: "/var/run/secrets/vault", + RunAsNonRoot: true, + RunAsUser: int64(1000), + RunAsGroup: int64(1000), }, }, wantedPod: &corev1.Pod{ From 595a2b05758c4147b950d333932128f1f5ad2e01 Mon Sep 17 00:00:00 2001 From: Mark Sagi-Kazar Date: Tue, 5 Sep 2023 17:28:12 +0200 Subject: [PATCH 4/4] fix: only set capabilities for agent templating when necessary Signed-off-by: Mark Sagi-Kazar --- pkg/webhook/pod.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkg/webhook/pod.go b/pkg/webhook/pod.go index 6573518c..1ea4f2b3 100644 --- a/pkg/webhook/pod.go +++ b/pkg/webhook/pod.go @@ -683,6 +683,13 @@ func getInitContainers(originalContainers []corev1.Container, podSecurityContext }) securityContext := getBaseSecurityContext(podSecurityContext, vaultConfig) + securityContext.Capabilities.Add = []corev1.Capability{ + "CHOWN", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + } containers = append(containers, corev1.Container{ Name: "vault-agent", @@ -787,8 +794,14 @@ func getAgentContainers(originalContainers []corev1.Container, podSecurityContex containers := []corev1.Container{} securityContext := getBaseSecurityContext(podSecurityContext, vaultConfig) - - securityContext.Capabilities.Add = append(securityContext.Capabilities.Add, "IPC_LOCK") + securityContext.Capabilities.Add = []corev1.Capability{ + "CHOWN", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "IPC_LOCK", + } if vaultConfig.AgentShareProcess { securityContext.Capabilities.Add = append(securityContext.Capabilities.Add, "SYS_PTRACE")