From ab6e215d7b5c2dac71195a0df2b9c182a03b8662 Mon Sep 17 00:00:00 2001 From: "Dmitry K. Anisimov" Date: Fri, 29 Nov 2024 00:08:00 +0200 Subject: [PATCH] feat: Add keychain provider for reading text/yaml/json secrets Signed-off-by: Dmitry K. Anisimov --- README.md | 93 ++++++++++++++++++++---------- go.mod | 1 + pkg/providers/keychain/keychain.go | 67 +++++++++++++++++++++ vals.go | 6 ++ 4 files changed, 138 insertions(+), 29 deletions(-) create mode 100644 pkg/providers/keychain/keychain.go diff --git a/README.md b/README.md index 5e239b4..157acfe 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ It supports various backends including: - HCP Vault Secrets - Bitwarden - HTTP JSON +- Keychain - Use `vals eval -f refs.yaml` to replace all the `ref`s in the file to actual values and secrets. - Use `vals exec -f env.yaml -- ` to populate envvars and execute the command. @@ -206,30 +207,53 @@ Please see the [relevant unit test cases](https://github.com/helmfile/vals/blob/ ## Supported Backends -- [Vault](#vault) -- [AWS SSM Parameter Store](#aws-ssm-parameter-store) -- [AWS Secrets Manager](#aws-secrets-manager) -- [AWS S3](#aws-s3) -- [GCP Secrets Manager](#gcp-secrets-manager) -- [GCP KMS](#gcp-kms) -- [Google Sheets](#google-sheets) -- [Google GCS](#google-gcs) -- [SOPS](#sops) powered by [sops](https://github.com/getsops/sops) -- [Terraform (tfstate)](#terraform-tfstate) powered by [tfstate-lookup](https://github.com/fujiwara/tfstate-lookup) -- [Echo](#echo) -- [File](#file) -- [Azure Key Vault](#azure-key-vault) -- [EnvSubst](#envsubst) -- [GitLab](#gitlab) -- [1Password](#1password) -- [1Password Connect](#1password-connect) -- [Doppler](#doppler) -- [Pulumi State](#pulumi-state) -- [Kubernetes](#kubernetes) -- [Conjur](#conjur) -- [HCP Vault Secrets](#hcp-vault-secrets) -- [HTTP JSON](#http-json) -- [Bitwarden](#bitwarden) +- [vals](#vals) + - [Usage](#usage) +- [CLI](#cli) + - [Helm](#helm) + - [Go](#go) + - [Expression Syntax](#expression-syntax) + - [Supported Backends](#supported-backends) + - [Vault](#vault) + - [Authentication](#authentication) + - [AWS](#aws) + - [AWS SSM Parameter Store](#aws-ssm-parameter-store) + - [AWS Secrets Manager](#aws-secrets-manager) + - [AWS S3](#aws-s3) + - [AWS KMS](#aws-kms) + - [Google GCS](#google-gcs) + - [GCP Secrets Manager](#gcp-secrets-manager) + - [GCP KMS](#gcp-kms) + - [Google Sheets](#google-sheets) + - [Terraform (tfstate)](#terraform-tfstate) + - [Terraform in GCS bucket (tfstategs)](#terraform-in-gcs-bucket-tfstategs) + - [Terraform in S3 bucket (tfstates3)](#terraform-in-s3-bucket-tfstates3) + - [Terraform in AzureRM Blob storage (tfstateazurerm)](#terraform-in-azurerm-blob-storage-tfstateazurerm) + - [Terraform in Terraform Cloud / Terraform Enterprise (tfstateremote)](#terraform-in-terraform-cloud-terraform-enterprise-tfstateremote) + - [SOPS](#sops) + - [Keychain](#keychain) + - [Echo](#echo) + - [File](#file) + - [Azure Key Vault](#azure-key-vault) + - [Authentication](#authentication-1) + - [EnvSubst](#envsubst) + - [GitLab Secrets](#gitlab-secrets) + - [1Password](#1password) + - [1Password Connect](#1password-connect) + - [Doppler](#doppler) + - [Pulumi State](#pulumi-state) + - [Kubernetes](#kubernetes) + - [Conjur](#conjur) + - [HCP Vault Secrets](#hcp-vault-secrets) + - [Bitwarden](#bitwarden) + - [HTTP JSON](#http-json) + - [Fetch string value](#fetch-string-value) + - [Fetch integer value](#fetch-integer-value) + - [Advanced Usages](#advanced-usages) + - [Discriminating config and secrets](#discriminating-config-and-secrets) + - [Non-Goals](#non-goals) + - [Complex String-Interpolation / Template Functions](#complex-string-interpolation-template-functions) + - [Merge](#merge) Please see [pkg/providers](https://github.com/helmfile/vals/tree/master/pkg/providers) for the implementations of all the providers. The package names corresponds to the URI schemes. @@ -592,6 +616,18 @@ Examples: - `ref+sops://path/to/file#/foo/bar` reads `path/to/file` as a `yaml` file and returns the value at `foo.bar`. - `ref+sops://path/to/file?format=json#/foo/bar` reads `path/to/file` as a `json` file and returns the value at `foo.bar`. +### Keychain + +Keychain provider is going to be available on macOS only. It reads a secret from the macOS Keychain. + +- `ref+echo://KEY1/[#/path/to/the/value]` + +Examples: + +- `security add-generic-password -U -a ${USER} -s "secret-name" -D "vals-secret" -w '{"foo":{"bar":"baz"}}'` - will create a secret in the Keychain with the name `secret-name` and the value `{"foo":{"bar":"baz"}}`, `vals-secret` is required to be able to find the secret in the Keychain. +- `echo 'foo: ref+keychain://secret-name' | vals eval -f -` - will read the secret from the Keychain with the name `secret-name` and replace the `foo` with the secret value. +- `echo 'foo: ref+keychain://secret-name#/foo/bar' | vals eval -f -` - will read the secret from the Keychain with the name `secret-name` and replace the `foo` with the value at the path `$.foo.bar`. + ### Echo Echo provider echoes the string for testing purpose. Please read [the original proposal](https://github.com/roboll/helmfile/pull/920#issuecomment-548213738) to get why we might need this. @@ -603,7 +639,6 @@ Examples: - `ref+echo://foo/bar` generates `foo/bar` - `ref+echo://foo/bar/baz#/foo/bar` generates `baz`. This works by the host and the path part `foo/bar/baz` generating an object `{"foo":{"bar":"baz"}}` and the fragment part `#/foo/bar` results in digging the object to obtain the value at `$.foo.bar`. - ### File File provider reads a local text file, or the value for the specific path in a YAML/JSON file. @@ -834,7 +869,7 @@ Example: ### Bitwarden -This provider retrieves the secrets stored in Bitwarden. It uses the [Bitwarden Vault-Management API](https://bitwarden.com/help/vault-management-api/) that is included in the [Bitwarden CLI](https://github.com/bitwarden/clients) by executing `bw serve`. +This provider retrieves the secrets stored in Bitwarden. It uses the [Bitwarden Vault-Management API](https://bitwarden.com/help/vault-management-api/) that is included in the [Bitwarden CLI](https://github.com/bitwarden/clients) by executing `bw serve`. Environment variables: @@ -857,7 +892,7 @@ Examples: This provider retrieves values stored in JSON hosted by a HTTP frontend. -This provider is built on top of [jsonquery](https://pkg.go.dev/github.com/antchfx/jsonquery@v1.3.3) and [xpath](https://pkg.go.dev/github.com/antchfx/xpath@v1.2.3) packages. +This provider is built on top of [jsonquery](https://pkg.go.dev/github.com/antchfx/jsonquery@v1.3.3) and [xpath](https://pkg.go.dev/github.com/antchfx/xpath@v1.2.3) packages. Given the diverse array of JSON structures that can be encountered, utilizing jsonquery with XPath presents a more effective approach for handling this variability in data structures. @@ -881,7 +916,7 @@ Let's say you want to fetch the below JSON object from https://api.github.com/us "name": "go-yaml" } ] -``` +``` ``` # To get name="chartify" using https protocol you would use: ref+httpjson://api.github.com/users/helmfile/repos#///*[1]/name @@ -904,7 +939,7 @@ Let's say you want to fetch the below JSON object from https://api.github.com/us "id": 251296379 } ] -``` +``` ``` # Running the following will return: 2.51296379e+08 ref+httpjson://api.github.com/users/helmfile/repos#///*[1]/id diff --git a/go.mod b/go.mod index 07e6c6c..e67bff5 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,7 @@ require ( github.com/google/go-jsonnet v0.20.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/ianlancetaylor/demangle v0.0.0-20240805132620-81f5be970eca // indirect + github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect diff --git a/pkg/providers/keychain/keychain.go b/pkg/providers/keychain/keychain.go new file mode 100644 index 0000000..7132ca0 --- /dev/null +++ b/pkg/providers/keychain/keychain.go @@ -0,0 +1,67 @@ +package keychain + +import ( + "errors" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/helmfile/vals/pkg/api" + "github.com/keybase/go-keychain" +) + +const keychainKind = "vals-secret" + +type provider struct { +} + +func New(cfg api.StaticConfig) *provider { + p := &provider{} + return p +} + +func getKeychainSecret(key string) ([]byte, error) { + query := keychain.NewItem() + query.SetSecClass(keychain.SecClassGenericPassword) + query.SetLabel(key) + query.SetDescription(keychainKind) + query.SetMatchLimit(keychain.MatchLimitOne) + query.SetReturnData(true) + + results, err := keychain.QueryItem(query) + if err != nil { + return nil, err + } else if len(results) == 0 { + return nil, errors.New("not found") + } + + return results[0].Data, nil +} + +func (p *provider) GetString(key string) (string, error) { + key = strings.TrimSuffix(key, "/") + key = strings.TrimSpace(key) + + secret, err := getKeychainSecret(key) + if err != nil { + return "", err + } + + return string(secret), err +} + +func (p *provider) GetStringMap(key string) (map[string]interface{}, error) { + key = strings.TrimSuffix(key, "/") + key = strings.TrimSpace(key) + + secret, err := getKeychainSecret(key) + if err != nil { + return nil, err + } + + m := map[string]interface{}{} + if err := yaml.Unmarshal(secret, &m); err != nil { + return nil, err + } + return m, nil +} diff --git a/vals.go b/vals.go index fdb23a7..54bc3de 100644 --- a/vals.go +++ b/vals.go @@ -37,6 +37,7 @@ import ( "github.com/helmfile/vals/pkg/providers/hcpvaultsecrets" "github.com/helmfile/vals/pkg/providers/httpjson" "github.com/helmfile/vals/pkg/providers/k8s" + "github.com/helmfile/vals/pkg/providers/keychain" "github.com/helmfile/vals/pkg/providers/onepassword" "github.com/helmfile/vals/pkg/providers/onepasswordconnect" "github.com/helmfile/vals/pkg/providers/pulumi" @@ -91,6 +92,7 @@ const ( ProviderTFStateRemote = "tfstateremote" ProviderAzureKeyVault = "azurekeyvault" ProviderEnvSubst = "envsubst" + ProviderKeychain = "keychain" ProviderOnePassword = "op" ProviderOnePasswordConnect = "onepasswordconnect" ProviderDoppler = "doppler" @@ -245,6 +247,9 @@ func (r *Runtime) prepare() (*expansion.ExpandRegexMatch, error) { case ProviderKms: p := awskms.New(conf) return p, nil + case ProviderKeychain: + p := keychain.New(conf) + return p, nil case ProviderEnvSubst: p := envsubst.New(conf) return p, nil @@ -491,6 +496,7 @@ var KnownValuesTypes = []string{ ProviderTFState, ProviderFile, ProviderEcho, + ProviderKeychain, ProviderEnvSubst, ProviderPulumiStateAPI, }