From c5f46418202131c13fa91b15b20952d78d424192 Mon Sep 17 00:00:00 2001 From: Richard Kovacs Date: Thu, 7 Jul 2022 06:45:12 +0200 Subject: [PATCH] Generate manifests for producation usage (#130) * Generate manifests for producation usage * Multiple configuration options for Vault * Fix generator test * Fix generator test * Fix generator test * Fix generator test --- .github/workflows/_e2e-test.yml | 2 - .github/workflows/generator-test-on-pr.yml | 81 ++++++++++++++ .gitignore | 3 +- .task/cluster.yml | 3 + .task/docker.yml | 20 ++-- .task/fetch.yml | 21 ++-- .task/go.yml | 38 +++++-- .task/prod.yml | 64 +++++++++++ Dockerfile | 3 +- README.md | 94 ++++++++++++++++ Taskfile.yml | 32 ++---- .../docker-compose.override.awskms.yaml | 11 ++ .../docker-compose.override.vault.yaml | 10 ++ deployment/docker-compose/docker-compose.yaml | 27 +++++ deployment/kustomize/configmap-awskms.yaml | 7 ++ .../kustomize/configmap-vault-generator.yaml | 34 ++++++ deployment/kustomize/configmap-vault.yaml | 42 ++++++++ deployment/kustomize/daemonset.yaml | 102 ++++++++++++++++++ deployment/kustomize/kustomization.yaml | 15 +++ deployment/kustomize/rbac.yaml | 17 +++ deployment/kustomize/sidecar-awskms.yaml | 59 ++++++++++ deployment/kustomize/sidecar-vault.yaml | 47 ++++++++ deployment/systemd/tousseau-awskms.service | 17 +++ deployment/systemd/tousseau-proxy.service | 17 +++ deployment/systemd/tousseau-trousseau.service | 18 ++++ deployment/systemd/tousseau-vault.service | 17 +++ localdev.md | 8 -- providers/awskms/localdev.md | 3 +- providers/awskms/main.go | 4 +- providers/awskms/pkg/awskms/awskms.go | 6 -- providers/awskms/pkg/awskms/config.go | 2 - providers/debug/main.go | 2 +- providers/vault/localdev.md | 2 +- providers/vault/main.go | 4 +- proxy/main.go | 4 +- scripts/generic/vault-kms-provider.yaml | 8 +- .../archives/k8s/encryption-config.yaml | 2 +- .../hcvault/archives/k8s/kube-apiserver.yaml | 6 +- .../archives/k8s/trousseau-hcvault.yaml | 8 +- scripts/hcvault/archives/rke2/config.yaml | 2 +- .../archives/rke2/trousseau-hcvault.yaml | 8 +- .../archives/testing/encryption-config.yaml | 2 +- scripts/hcvault/archives/testing/kms.yaml | 4 +- .../trousseau-hcvault-daemonset.yaml | 8 +- .../trousseau-hcvault-kube-apiserver.yaml | 2 +- .../kuttl/kube-v1.22/encryption-config.yaml | 2 +- tests/e2e/kuttl/kube-v1.22/kind.yaml | 2 +- .../kuttl/kube-v1.23/encryption-config.yaml | 2 +- tests/e2e/kuttl/kube-v1.23/kind.yaml | 2 +- .../kuttl/kube-v1.24/encryption-config.yaml | 2 +- tests/e2e/kuttl/kube-v1.24/kind.yaml | 2 +- trousseau/main.go | 6 +- 52 files changed, 788 insertions(+), 116 deletions(-) create mode 100644 .github/workflows/generator-test-on-pr.yml create mode 100644 .task/prod.yml create mode 100644 deployment/docker-compose/docker-compose.override.awskms.yaml create mode 100644 deployment/docker-compose/docker-compose.override.vault.yaml create mode 100644 deployment/docker-compose/docker-compose.yaml create mode 100644 deployment/kustomize/configmap-awskms.yaml create mode 100644 deployment/kustomize/configmap-vault-generator.yaml create mode 100644 deployment/kustomize/configmap-vault.yaml create mode 100644 deployment/kustomize/daemonset.yaml create mode 100644 deployment/kustomize/kustomization.yaml create mode 100644 deployment/kustomize/rbac.yaml create mode 100644 deployment/kustomize/sidecar-awskms.yaml create mode 100644 deployment/kustomize/sidecar-vault.yaml create mode 100644 deployment/systemd/tousseau-awskms.service create mode 100644 deployment/systemd/tousseau-proxy.service create mode 100644 deployment/systemd/tousseau-trousseau.service create mode 100644 deployment/systemd/tousseau-vault.service diff --git a/.github/workflows/_e2e-test.yml b/.github/workflows/_e2e-test.yml index 75ef6a3..e063452 100644 --- a/.github/workflows/_e2e-test.yml +++ b/.github/workflows/_e2e-test.yml @@ -25,8 +25,6 @@ jobs: - uses: actions/checkout@v2 - name: install Taskfile run: mkdir bin && cd bin ; curl -Ls https://github.com/go-task/task/releases/download/v3.13.0/task_linux_amd64.tar.gz | tar -xz task - - name: fetch dependencies - run: ./bin/task fetch:kuttl fetch:kind - name: e2e test run: PATH=./bin:$PATH ./bin/task go:e2e-tests:${{ inputs.provider }} KIND_CLUSTER_VERSION=${{ inputs.kubever }} diff --git a/.github/workflows/generator-test-on-pr.yml b/.github/workflows/generator-test-on-pr.yml new file mode 100644 index 0000000..a343ab9 --- /dev/null +++ b/.github/workflows/generator-test-on-pr.yml @@ -0,0 +1,81 @@ +name: Generator test on pr + +on: + pull_request: + branches: [ main, v2* ] + workflow_dispatch: + +permissions: + contents: read + pull-requests: read + actions: read + security-events: write + packages: write + +concurrency: + group: ci-generator-${{ github.ref }}-1 + cancel-in-progress: true + +jobs: + docker-compose: + name: docker compose + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: install Taskfile + run: mkdir bin && cd bin ; curl -Ls https://github.com/go-task/task/releases/download/v3.13.0/task_linux_amd64.tar.gz | tar -xz task + - uses: "finnp/create-file-action@master" + env: + FILE_NAME: "trousseau-env" + FILE_DATA: | + TR_VERBOSE_LEVEL=3 + TR_ENABLED_PROVIDERS="--enabled-providers=awskms --enabled-providers=vault" + TR_SOCKET_LOCATION=${PWD}/bin/run + TR_PROXY_IMAGE=ondat/trousseau:proxy-develop + TR_AWSKMS_IMAGE=ondat/trousseau:awskms-develop + TR_VAULT_IMAGE=ondat/trousseau:vault-develop + TR_TROUSSEAU_IMAGE=ondat/trousseau:trousseau-develop + TR_AWSKMS_CREDENTIALS=${HOME}/.aws/credentials + TR_AWSKMS_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/awskms.yaml + TR_VAULT_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/vault.yaml + - name: generate services + run: PATH=./bin:$PATH ./bin/task prod:generate:docker-compose ENV_LOCATION=trousseau-env + - name: validate compose files + run: cd generated_manifests/docker-compose ; docker compose -f docker-compose.yaml -f docker-compose.override.awskms.yaml -f docker-compose.override.vault.yaml config + kustomize: + name: kustomize + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: install Taskfile + run: mkdir bin && cd bin ; curl -Ls https://github.com/go-task/task/releases/download/v3.13.0/task_linux_amd64.tar.gz | tar -xz task + - uses: "finnp/create-file-action@master" + env: + FILE_NAME: "awskms.yaml" + FILE_DATA: | + profile: default + - uses: "finnp/create-file-action@master" + env: + FILE_NAME: "trousseau-env" + FILE_DATA: | + TR_VERBOSE_LEVEL=3 + TR_ENABLED_PROVIDERS="--enabled-providers=awskms --enabled-providers=vault" + TR_SOCKET_LOCATION=${PWD}/bin/run + TR_PROXY_IMAGE=ondat/trousseau:proxy-develop + TR_AWSKMS_IMAGE=ondat/trousseau:awskms-develop + TR_VAULT_IMAGE=ondat/trousseau:vault-develop + TR_TROUSSEAU_IMAGE=ondat/trousseau:trousseau-develop + TR_AWSKMS_CREDENTIALS=${HOME}/.aws/credentials + TR_AWSKMS_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/awskms.yaml + TR_VAULT_CONFIG=${PWD}/tests/e2e/kuttl/kube-v1.24/vault.yaml + TR_VAULT_ADDRESS=http://127.0.0.1:8200 + - name: generate services + run: PATH=./bin:$PATH ./bin/task prod:generate:kustomize ENV_LOCATION=trousseau-env + - uses: karancode/kustomize-github-action@master + with: + kustomize_version: '4.5.5' + kustomize_build_dir: 'generated_manifests/kustomize' + kustomize_output_file: "manifests.yaml" + - uses: makocchi-git/actions-k8s-manifests-validate-kubeval@master + with: + files: manifests.yaml diff --git a/.gitignore b/.gitignore index ec30f49..aa0e6e4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ bin *.socket cover.out kind-logs-*/ -kubeconfig \ No newline at end of file +kubeconfig +generated_manifests/ \ No newline at end of file diff --git a/.task/cluster.yml b/.task/cluster.yml index 8bd2702..66c5eb9 100644 --- a/.task/cluster.yml +++ b/.task/cluster.yml @@ -4,9 +4,12 @@ tasks: desc: create kind cluster deps: - delete + - :fetch:kind cmds: - ./bin/kind create cluster --retain --name "{{.KIND_CLUSTER_NAME}}" --wait 2m --config ./tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/kind.yaml delete: desc: destroy kind cluster + deps: + - :fetch:kind cmds: - ./bin/kind delete cluster --name "{{.KIND_CLUSTER_NAME}}" \ No newline at end of file diff --git a/.task/docker.yml b/.task/docker.yml index 76acbf9..ebe4295 100644 --- a/.task/docker.yml +++ b/.task/docker.yml @@ -71,19 +71,19 @@ tasks: - task: run:trousseau run:proxy: deps: - - :bin-dir:init + - :run-dir:init cmds: - docker rm -f trousseau-proxy || true - - docker run -d --name trousseau-proxy --rm -v $PWD/bin/run:/opt/vault-kms $DOCKER_REGISTRY/$IMAGE_NAME:proxy-$IMAGE_VERSION + - docker run -d --name trousseau-proxy --rm -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:proxy-$IMAGE_VERSION run:debug: deps: - - :bin-dir:init + - :run-dir:init cmds: - docker rm -f trousseau-debug || true - - docker run -d --name trousseau-debug --rm -v $PWD/bin/run:/opt/vault-kms $DOCKER_REGISTRY/$IMAGE_NAME:debug-$IMAGE_VERSION + - docker run -d --name trousseau-debug --rm -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:debug-$IMAGE_VERSION run:vault: deps: - - :bin-dir:init + - :run-dir:init cmds: - docker rm -f trousseau-local-vault || true - docker run -d --name=trousseau-local-vault --cap-add=IPC_LOCK -e 'VAULT_DEV_ROOT_TOKEN_ID=vault-kms-demo' vault @@ -91,20 +91,20 @@ tasks: - docker exec -e VAULT_ADDR=http://127.0.0.1:8200 trousseau-local-vault vault login vault-kms-demo - docker exec -e VAULT_ADDR=http://127.0.0.1:8200 trousseau-local-vault vault secrets enable transit - docker rm -f trousseau-vault || true - - docker run -d --name trousseau-vault --rm --network=container:trousseau-local-vault -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/vault.yaml:/etc/config.yaml -v $PWD/bin/run:/opt/vault-kms $DOCKER_REGISTRY/$IMAGE_NAME:vault-$IMAGE_VERSION --config-file-path=/etc/config.yaml -v=3 + - docker run -d --name trousseau-vault --rm --network=container:trousseau-local-vault -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/vault.yaml:/etc/config.yaml -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:vault-$IMAGE_VERSION --config-file-path=/etc/config.yaml -v=3 run:awskms: deps: - - :bin-dir:init + - :run-dir:init cmds: - docker rm -f trousseau-local-aws || true - docker run --name trousseau-local-aws --rm --hostname localhost.localstack.cloud -d -e SERVICES=kms -e HOSTNAME=localhost.localstack.cloud -e HOSTNAME_EXTERNAL=localhost.localstack.cloud -e DEFAULT_REGION=eu-west-1 -e KMS_PROVIDER=kms-local -p 4566:4566 -p 4510-4559:4510-4559 localstack/localstack:0.14.4 - sleep 5 - 'printf %"s\n" "endpoint: https://localhost.localstack.cloud:4566" "profile: trousseau-local-aws" "keyArn: $(docker exec trousseau-local-aws awslocal kms create-key | grep Arn | cut -d''"'' -f4)" > tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/awskms.yaml' - docker rm -f trousseau-awskms || true - - docker run -d --name trousseau-awskms --rm --network=container:trousseau-local-aws -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/aws-credentials.ini:/.aws/credentials -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/awskms.yaml:/etc/config.yaml -v $PWD/bin/run:/opt/vault-kms $DOCKER_REGISTRY/$IMAGE_NAME:awskms-$IMAGE_VERSION --config-file-path=/etc/config.yaml -v=3 + - docker run -d --name trousseau-awskms --rm --network=container:trousseau-local-aws -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/aws-credentials.ini:/.aws/credentials -v $PWD/tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/awskms.yaml:/etc/config.yaml -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:awskms-$IMAGE_VERSION --config-file-path=/etc/config.yaml -v=3 run:trousseau: deps: - - :bin-dir:init + - :run-dir:init cmds: - docker rm -f trousseau-core || true - - docker run -d --name trousseau-core --rm -v $PWD/bin/run:/opt/vault-kms $DOCKER_REGISTRY/$IMAGE_NAME:trousseau-$IMAGE_VERSION {{.ENABLED_PROVIDERS}} -v=3 + - docker run -d --name trousseau-core --rm -v $PWD/bin/run:/opt/trousseau-kms $DOCKER_REGISTRY/$IMAGE_NAME:trousseau-$IMAGE_VERSION {{.ENABLED_PROVIDERS}} -v=3 diff --git a/.task/fetch.yml b/.task/fetch.yml index cf6aba5..955beed 100644 --- a/.task/fetch.yml +++ b/.task/fetch.yml @@ -7,14 +7,11 @@ vars: VAULT_VERSION: 1.8.1 KUBECTL_VERSION: 1.21.1 KUTTL_VERSION: 0.12.1 + ENVSUBST_VERSION: 1.2.0 HUSKY_VERSION: 0.2.8 + CAP_ARCH: + sh: uname tasks: - all: - desc: fetch all tools - cmds: - - task: kubectl - - task: kind - - task: kuttl golangci: deps: - :bin-dir:init @@ -83,6 +80,15 @@ tasks: - cd bin ; curl -L https://github.com/kudobuilder/kuttl/releases/download/v{{.KUTTL_VERSION}}/kuttl_{{.KUTTL_VERSION}}_{{OS}}_x86_64.tar.gz | tar -xz kubectl-kuttl status: - test -f ./bin/kuttl + envsubst: + deps: + - :bin-dir:init + desc: install envsubst + cmds: + - cd bin ; curl -o envsubst -L https://github.com/a8m/envsubst/releases/download/v{{.ENVSUBST_VERSION}}/envsubst-{{.CAP_ARCH}}-x86_64 + - chmod +x ./bin/envsubst + status: + - test -f ./bin/envsubst husky: deps: - :bin-dir:init @@ -91,8 +97,5 @@ tasks: - cd bin ; curl -L https://github.com/automation-co/husky/releases/download/v{{.HUSKY_VERSION}}/husky_{{.HUSKY_VERSION}}_{{.CAP_ARCH}}_x86_64.tar.gz | tar -xz husky - chmod +x ./bin/husky - ./bin/husky install - vars: - CAP_ARCH: - sh: uname status: - test -f ./bin/husky \ No newline at end of file diff --git a/.task/go.yml b/.task/go.yml index 84004a3..bf1fef6 100644 --- a/.task/go.yml +++ b/.task/go.yml @@ -95,22 +95,32 @@ tasks: - task: gosec:trousseau gosec:proxy: dir: proxy + deps: + - :fetch:gosec cmds: - gosec ./... gosec:debug: dir: providers/debug + deps: + - :fetch:gosec cmds: - gosec ./... gosec:vault: dir: providers/vault + deps: + - :fetch:gosec cmds: - gosec ./... gosec:awskms: dir: providers/awskms + deps: + - :fetch:gosec cmds: - gosec ./... gosec:trousseau: dir: trousseau + deps: + - :fetch:gosec cmds: - gosec ./... golangci: @@ -123,14 +133,20 @@ tasks: - task: golangci:trousseau golangci:proxy: dir: proxy + deps: + - :fetch:golangci cmds: - golangci-lint run -c ../.golangci.yaml golangci:debug: dir: providers/debug + deps: + - :fetch:golangci cmds: - golangci-lint run -c ../../.golangci.yaml golangci:vault: dir: providers/vault + deps: + - :fetch:golangci cmds: - golangci-lint run -c ../../.golangci.yaml golangci:awskms: @@ -139,6 +155,8 @@ tasks: - golangci-lint run -c ../../.golangci.yaml golangci:trousseau: dir: trousseau + deps: + - :fetch:golangci cmds: - golangci-lint run -c ../.golangci.yaml unit-tests: @@ -172,35 +190,35 @@ tasks: run:proxy: dir: proxy deps: - - :bin-dir:init + - :run-dir:init - tidy:proxy cmds: - go run main.go --listen-addr unix://../bin/run/proxy.socket --trousseau-addr ../bin/run/trousseau.socket run:debug: dir: providers/debug deps: - - :bin-dir:init + - :run-dir:init - tidy:debug cmds: - go run main.go --listen-addr unix://../../bin/run/debug/debug.socket run:vault: dir: providers/vault deps: - - :bin-dir:init + - :run-dir:init - tidy:vault cmds: - go run -ldflags '-X github.com/ondat/trousseau/pkg/utils.SecretLogDivider=1' main.go --config-file-path ../../tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/vault.yaml --listen-addr unix://../../bin/run/vault/vault.socket --zap-encoder=console --v=5 run:awskms: dir: providers/awskms deps: - - :bin-dir:init + - :run-dir:init - tidy:awskms cmds: - go run -ldflags '-X github.com/ondat/trousseau/pkg/utils.SecretLogDivider=1' main.go --config-file-path ../../tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/awskms.yaml --listen-addr unix://../../bin/run/awskms/awskms.socket --zap-encoder=console --v=5 run:trousseau: dir: trousseau deps: - - :bin-dir:init + - :run-dir:init - tidy:trousseau cmds: - go run -ldflags '-X github.com/ondat/trousseau/pkg/utils.SecretLogDivider=1' main.go {{.ENABLED_PROVIDERS}} --socket-location ../bin/run --listen-addr unix://../bin/run/trousseau.socket --zap-encoder=console --v=5 @@ -212,30 +230,36 @@ tasks: - task: e2e-tests:awskms e2e-tests:vault: deps: + - :fetch:kuttl + - :fetch:kind - :docker:build:proxy - :docker:build:vault - :docker:build:trousseau cmds: - task: :docker:run:proxy - task: :docker:run:vault - - ENABLED_PROVIDERS="--enabled-providers vault" task docker:run:trousseau + - ENABLED_PROVIDERS="--enabled-providers=vault" task docker:run:trousseau - task: :cluster:create - ./bin/kubectl-kuttl test --config tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/kuttl.yaml - task: :cluster:delete e2e-tests:awskms: deps: + - :fetch:kuttl + - :fetch:kind - :docker:build:proxy - :docker:build:awskms - :docker:build:trousseau cmds: - task: :docker:run:proxy - task: :docker:run:awskms - - ENABLED_PROVIDERS="--enabled-providers awskms" task docker:run:trousseau + - ENABLED_PROVIDERS="--enabled-providers=awskms" task docker:run:trousseau - task: :cluster:create - ./bin/kubectl-kuttl test --config tests/e2e/kuttl/kube-v{{.KIND_CLUSTER_VERSION}}/kuttl.yaml - task: :cluster:delete e2e-tests:debug: deps: + - :fetch:kuttl + - :fetch:kind - :docker:build:proxy - :docker:build:debug - :docker:build:trousseau diff --git a/.task/prod.yml b/.task/prod.yml new file mode 100644 index 0000000..65a7288 --- /dev/null +++ b/.task/prod.yml @@ -0,0 +1,64 @@ +version: 3 +silent: true +vars: + SCRIPT: scripts/hcvault/archives/testing + ENV_LOCATION: '{{.ENV_LOCATION | default "/please/set/ENV_LOCATION"}}' +tasks: + gen-dir:init: + desc: create bin directory + cmds: + - mkdir -p generated_manifests + status: + - test -d generated_manifests + prometheus:deploy: + deps: + - task: :fetch:helm + - task: :fetch:kubectl + desc: install prometheus and grafana on cluster + cmds: + - ./bin/helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + - ./bin/helm upgrade --install prometheus prometheus-community/kube-prometheus-stack --wait + - ./bin/kubectl apply -f {{.SCRIPT}}/prometheus.yaml --wait + - ./bin/kubectl apply -f {{.SCRIPT}}/grafana-dashboard.yaml --wait + prometheus:port-forward: + deps: + - task: :fetch:kubectl + desc: port forwarding for prometheus + cmds: + - ./bin/kubectl port-forward svc/prometheus-kube-prometheus-prometheus 9090 + grafana:port-forward: + deps: + - task: :fetch:kubectl + desc: open grafana (admin/prom-operator) + cmds: + - ./bin/kubectl port-forward svc/prometheus-grafana 8300:80 + generate:systemd: + desc: generate systemd services + deps: + - gen-dir:init + cmds: + - rm -rf generated_manifests/systemd/* + - cp -rf deployment/systemd generated_manifests + - cp -f {{.ENV_LOCATION}} generated_manifests/systemd/trousseau-env + generate:docker-compose: + desc: generate docker compose services + deps: + - gen-dir:init + - :fetch:envsubst + cmds: + - mkdir -p generated_manifests/docker-compose ; rm -rf generated_manifests/docker-compose/* + - source {{.ENV_LOCATION}} ; + export $(echo "${!TR_*}") ; + for f in `cd deployment ; find docker-compose -type f`; do ./bin/envsubst -no-empty -i deployment/$f -o generated_manifests/$f; done + generate:kustomize: + desc: generate kustomize manifests + deps: + - gen-dir:init + - :fetch:envsubst + cmds: + - mkdir -p generated_manifests/kustomize ; rm -rf generated_manifests/kustomize/* + - source {{.ENV_LOCATION}} ; + TR_ENABLED_PROVIDERS=$(echo ${TR_ENABLED_PROVIDERS} | sed "s/ --/\n - --/") ; + test -n "${TR_AWSKMS_CONFIG}" && TR_AWSKMS_CONFIG=$(cat ${TR_AWSKMS_CONFIG} 2>/dev/null | sed 's/^/ /') ; + export $(echo "${!TR_*}") ; + for f in `cd deployment ; find kustomize -type f`; do ./bin/envsubst -no-empty -i deployment/$f -o generated_manifests/$f; done diff --git a/Dockerfile b/Dockerfile index 8865fe4..436d630 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,5 +22,6 @@ COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certifica COPY --from=build /work/${PROJECT}/endpoint /bin/ -USER 10123 +USER 10123:10123 + ENTRYPOINT ["/bin/endpoint"] diff --git a/README.md b/README.md index 88e5f1e..48082cd 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,100 @@ The name ***trousseau*** comes from the French language and is usually associate The following blog post provides an overview of a production use case for a Hong Kong based Service Provider leveraging Suse, RKE2, HashiCorp Vault and Trousseau to secure their workload hosted for Government agencies: * https://www.ondat.io/news/trousseau-open-source-project-made-available-to-add-security-in-kubernetes +### Run Trousseau in production +Clone the repo and create your environment file: +```bash +TR_VERSION=31b93747fc4fd438a6b30de70ff16d4a45271366 +TR_VERBOSE_LEVEL=1 +TR_SOCKET_LOCATION=/opt/trousseau-kms +TR_PROXY_IMAGE=ghcr.io/ondat/trousseau:proxy-${TR_VERSION} +TR_TROUSSEAU_IMAGE=ghcr.io/ondat/trousseau:trousseau-${TR_VERSION} +# Please configure your KMS plugins +TR_ENABLED_PROVIDERS="--enabled-providers=awskms --enabled-providers=vault" +TR_AWSKMS_IMAGE=ghcr.io/ondat/trousseau:awskms-${TR_VERSION} +TR_AWSKMS_CONFIG=awskms.yaml # For Kubernetes, file must exists only for generation +TR_AWSKMS_CREDENTIALS=.aws/credentials +TR_VAULT_IMAGE=ghcr.io/ondat/trousseau:vault-${TR_VERSION} +TR_VAULT_ADDRESS=https://127.0.0.1:8200 +TR_VAULT_CONFIG=vault.yaml +``` + +Create shared items on target host: +```bash +mkdir -p $TR_SOCKET_LOCATION +sudo chown 10123:10123 $TR_SOCKET_LOCATION +sudo chown 10123:10123 $TR_AWSKMS_CREDENTIALS +# On case you disabled Vault agen config generation +sudo chown 10123:10123 $TR_VAULT_CONFIG +``` + +Create your config files: +```yaml +# awskms.yaml +profile: profile +keyArn: keyArn +# Optional fields +roleArn: roleArn +encryptionContext: + foo: bar +``` +```yaml +# vault.yaml +keyNames: +- keyNames +address: address +token: token +``` + +Generate service files or manifests: +```bash +task prod:generate:systemd ENV_LOCATION=./bin/trousseau-env +task prod:generate:docker-compose ENV_LOCATION=./bin/trousseau-env +task prod:generate:kubernetes ENV_LOCATION=./bin/trousseau-env +``` + +Verify output: +```bash +ls -l generated_manifests/systemd +ls -l generated_manifests/docker-compose +ls -l generated_manifests/kubernetes +``` + +Deploy the application and configure encryption: +``` +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: + - resources: + - secrets + providers: + - kms: + name: vaultprovider + endpoint: unix:///opt/trousseau-kms/proxy.socket + cachesize: 1000 + - identity: {} +``` + +Reconfigure Kubernetes API server: +``` +kind: ClusterConfiguration +apiServer: + extraArgs: + encryption-provider-config: "/etc/kubernetes/encryption-config.yaml" + extraVolumes: + - name: encryption-config + hostPath: "/etc/kubernetes/encryption-config.yaml" + mountPath: "/etc/kubernetes/encryption-config.yaml" + readOnly: true + pathType: File + - name: sock-path + hostPath: "/opt/trousseau-kms" + mountPath: "/opt/trousseau-kms" +``` + +Finally restart Kubernetes API server. + + ## Roadmap The roadmap items are described within [user story 50](https://github.com/ondat/trousseau/issues/50) Trousseau's roadmap milestone for v2 [here](https://github.com/orgs/ondat/projects/1/views/4](https://github.com/ondat/trousseau/milestone/2). diff --git a/Taskfile.yml b/Taskfile.yml index 8f1d21e..6436705 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -3,14 +3,14 @@ vars: KIND_CLUSTER_NAME: kms-vault KIND_CLUSTER_VERSION: 1.24 ENABLED_PROVIDERS: - sh: '([ -z "$ENABLED_PROVIDERS" ] && echo --enabled-providers debug) || echo $ENABLED_PROVIDERS' - SCRIPT: scripts/hcvault/archives/testing + sh: '([ -z "$ENABLED_PROVIDERS" ] && echo --enabled-providers=debug) || echo $ENABLED_PROVIDERS' silent: true includes: cluster: .task/cluster.yml docker: .task/docker.yml fetch: .task/fetch.yml go: .task/go.yml + prod: .task/prod.yml tasks: default: cmds: @@ -19,38 +19,20 @@ tasks: desc: create bin directory cmds: - mkdir -p ./bin + status: + - test -d ./bin + run-dir:init: + desc: create bin directory + cmds: - mkdir -pm 777 bin/run - mkdir -pm 777 bin/run/debug - mkdir -pm 777 bin/run/vault - mkdir -pm 777 bin/run/awskms status: - - test -d ./bin - test -d ./bin/run - test -d ./bin/run/debug - test -d ./bin/run/vault - test -d ./bin/run/awskms - prometheus:deploy: - deps: - - task: fetch:helm - - task: fetch:kubectl - desc: install prometheus and grafana on cluster - cmds: - - ./bin/helm repo add prometheus-community https://prometheus-community.github.io/helm-charts - - ./bin/helm upgrade --install prometheus prometheus-community/kube-prometheus-stack --wait - - ./bin/kubectl apply -f {{.SCRIPT}}/prometheus.yaml --wait - - ./bin/kubectl apply -f {{.SCRIPT}}/grafana-dashboard.yaml --wait - prometheus:port-forward: - deps: - - task: fetch:kubectl - desc: port forwarding for prometheus - cmds: - - ./bin/kubectl port-forward svc/prometheus-kube-prometheus-prometheus 9090 - grafana:port-forward: - deps: - - task: fetch:kubectl - desc: open grafana (admin/prom-operator) - cmds: - - ./bin/kubectl port-forward svc/prometheus-grafana 8300:80 example:load: desc: load demo data cmds: diff --git a/deployment/docker-compose/docker-compose.override.awskms.yaml b/deployment/docker-compose/docker-compose.override.awskms.yaml new file mode 100644 index 0000000..4b21ccb --- /dev/null +++ b/deployment/docker-compose/docker-compose.override.awskms.yaml @@ -0,0 +1,11 @@ +services: + awskms: + image: ${TR_AWSKMS_IMAGE} + command: --listen-addr=unix:///sockets/awskms/awskms.socket --config-file-path=/etc/config.yaml -v=${TR_VERBOSE_LEVEL} + volumes: + - sockets:/sockets:rw + - ${TR_AWSKMS_CREDENTIALS}:/.aws/credentials:r + - ${TR_AWSKMS_CONFIG}:/etc/config.yaml:r + restart: always + depends_on: + - init diff --git a/deployment/docker-compose/docker-compose.override.vault.yaml b/deployment/docker-compose/docker-compose.override.vault.yaml new file mode 100644 index 0000000..396debd --- /dev/null +++ b/deployment/docker-compose/docker-compose.override.vault.yaml @@ -0,0 +1,10 @@ +services: + vault: + image: ${TR_VAULT_IMAGE} + command: --listen-addr=unix:///sockets/vault/vault.socket --config-file-path=/etc/config.yaml -v=${TR_VERBOSE_LEVEL} + volumes: + - sockets:/sockets:rw + - ${TR_VAULT_CONFIG}:/etc/config.yaml:r + restart: always + depends_on: + - init diff --git a/deployment/docker-compose/docker-compose.yaml b/deployment/docker-compose/docker-compose.yaml new file mode 100644 index 0000000..36d9c4d --- /dev/null +++ b/deployment/docker-compose/docker-compose.yaml @@ -0,0 +1,27 @@ +services: + init: + image: busybox:stable-glibc + command: sh -c 'mkdir -p /sockets/awskms /sockets/vault /sockets/trousseau ; chown -R 10123:10123 /sockets/*' + volumes: + - sockets:/sockets:rw + proxy: + image: ${TR_PROXY_IMAGE} + command: --listen-addr=unix:///opt/trousseau-kms/proxy.socket --trousseau-addr=/sockets/trousseau/trousseau.socket + volumes: + - sockets:/sockets:rw + - ${TR_SOCKET_LOCATION}:/opt/trousseau-kms:rw + restart: always + depends_on: + - trousseau + trousseau: + image: ${TR_TROUSSEAU_IMAGE} + command: --listen-addr=unix:///sockets/trousseau/trousseau.socket --socket-location=/sockets ${TR_ENABLED_PROVIDERS} -v=${TR_VERBOSE_LEVEL} + volumes: + - sockets:/sockets:rw + restart: always + depends_on: + - init + - awskms + - vault +volumes: + sockets: {} diff --git a/deployment/kustomize/configmap-awskms.yaml b/deployment/kustomize/configmap-awskms.yaml new file mode 100644 index 0000000..f98406b --- /dev/null +++ b/deployment/kustomize/configmap-awskms.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: trousseau-awskms-config +data: + config.yaml: | +${TR_AWSKMS_CONFIG} diff --git a/deployment/kustomize/configmap-vault-generator.yaml b/deployment/kustomize/configmap-vault-generator.yaml new file mode 100644 index 0000000..5132abc --- /dev/null +++ b/deployment/kustomize/configmap-vault-generator.yaml @@ -0,0 +1,34 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: trousseau-kms-provider +spec: + template: + spec: + initContainers: + - name: vault-agent + image: vault + args: + - agent + - -config=/etc/vault/vault-agent-config.hcl + - -log-level=debug + env: + - name: VAULT_ADDR + value: ${TR_VAULT_ADDRESS} + volumeMounts: + - name: vault-agent-config + mountPath: /etc/vault + readOnly: true + - name: vault-config + mountPath: ${TR_VAULT_CONFIG} + volumes: + - configMap: + items: + - key: vault-agent-config.hcl + path: vault-agent-config.hcl + name: trousseau-vault-agent-config + name: vault-agent-config + - emptyDir: {} + hostPath: + name: vault-config + \ No newline at end of file diff --git a/deployment/kustomize/configmap-vault.yaml b/deployment/kustomize/configmap-vault.yaml new file mode 100644 index 0000000..b120267 --- /dev/null +++ b/deployment/kustomize/configmap-vault.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: trousseau-vault-agent-config +data: + vault-agent-config.hcl: | + exit_after_auth = true + pid_file = "/home/vault/pidfile" + auto_auth { + method "kubernetes" { + mount_path = "auth/kubernetes" + config = { + role = "trousseau" + } + } + sink "file" { + config = { + path = "/home/vault/.vault-token" + } + } + } + + template { + destination = "${TR_VAULT_CONFIG}" + contents = <