diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..eb398e4 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,22 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Package", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "subst/main.go", + "args": [ + "render", + "-v", + "debug", + "--skip-decrypt", + "/Users/foo/cluster" + ], + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md index eaa9641..e0a3422 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,13 @@ ![subst](./img/subst.png "subst") -__Currently under development and testing. I don't take any responsability for unexpected behavior. You know what you are doing :3__ - A simple extension over kustomize, which allows further variable substitution and introduces simplified yet strong secrets management (for multi tenancy use-cases). Extends to functionality of kustomize for ArgoCD users. # Functionality -The idea for subst is to act as complementary for kustomize. You can reference additional variables for your environment or from different kustomize paths, which are then accesible across your entire kustomize build. The kustomize you are referencing to is resolved (it's paths). In each of these paths you can create new substition files, which contain variables or secrets, which then can be used by your kustomization. The final output is your built kustomization with the substitutions made. +The idea for subst is to act as complementary for kustomize. You can reference additional variables for your environment or from different kustomize paths, which are then accesible across your entire kustomize build. The kustomize you are referencing to is resolved (it's paths). In each of these paths you can create new substitution files, which contain variables or secrets, which then can be used by your kustomization. The final output is your built kustomization with the substitutions made. -By default the all files are considered using this regex `(.*subst\.yaml|.*(ejson|vars))`. You can change the regex using: +By default the all files are considered using this regex `(subst\.yaml|.*(ejson))`. You can change the regex using: ``` subst render . --file-regex "custom-values\\.yaml" @@ -133,21 +131,21 @@ Note that directories do not resolve by recursion (eg. `/test/build/` only colle ### Environment -for environment variables which come from an argo application (`^ARGOCD_ENV_`) we remove the `ARGOCD_ENV_` and they are then available in your substitutions without the `ARGOCD_ENV_` prefix. This way they have the same name you have given them on the application ([Read More](https://argo-cd.readthedocs.io/en/stable/operator-manual/config-management-plugins/#using-environment-variables-in-your-plugin)). All the substions are available as flat key, so where needed you can use environment substitution. +For environment variables which come from an argo application (`^ARGOCD_ENV_`) we remove the `ARGOCD_ENV_` and they are then available in your substitutions without the `ARGOCD_ENV_` prefix. This way they have the same name you have given them on the application ([Read More](https://argo-cd.readthedocs.io/en/stable/operator-manual/config-management-plugins/#using-environment-variables-in-your-plugin)). All the substitutions are available as flat key, so where needed you can use environment substitution. ## Spruce -[Spruce](https://github.com/geofffranks/spruce) is used to access the substition variables, it has more flexability than [envsubst](#environment-substitution). You can grab values from the available substitutions using [Spruce Operators](https://github.com/geofffranks/spruce/blob/main/doc/operators.md). Spurce is greate, because it's operators are valid YAML which allows to build the kustomize without any further hacking. +[Spruce](https://github.com/geofffranks/spruce) is used to access the substitution variables, it has more flexability than [envsubst](#environment-substitution). You can grab values from the available substitutions using [Spruce Operators](https://github.com/geofffranks/spruce/blob/main/doc/operators.md). Spurce is great, because it's operators are valid YAML which allows to build the kustomize without any further hacking. ## Secrets -You can both encrypt files which are part of the kustomize build or which are used for substitution. Currently for secret decryption we support both [ejson](https://github.com/Shopify/ejson) and [sops](https://github.com/mozilla/sops). You can use any combination of these decryption providers together. The principal for all decryption provider is, that they should load the private keys while a substiution build is made instead of having a permanent keystore. This allows for secret tenancy (eg. one secret per argo application). The private keys are loaded from kubernetes secrets, therefor the plugin also creates it's own kubeconfig. +You can both encrypt files which are part of the kustomize build or which are used for substitution. Currently for secret decryption we support [ejson](https://github.com/Shopify/ejson). The principal for the decryption provider is, that it should load the private keys while a substitution build is made instead of having a permanent keystore. This allows for secret tenancy (eg. one secret per argo application). The private keys are loaded from kubernetes secrets, therefor the plugin also creates it's own kubeconfig. -The secrets are loaded based on the environment variables `$ARGOCD_APP_NAME` and `$ARGOCD_APP_NAMESPACE` are used. If an application is in a project, the value of `$ARGOCD_APP_NAME` looks like this: `_`. For example, if the application `my-app` is in the project `my-project`, the value of `$ARGOCD_APP_NAME` is `my-project_my-app`. All special characters within are converted to `-` (dash). For example, if the application `my-app` is in the project `my-project`, the value of `$ARGOCD_APP_NAME` is `my-project-my-app`. So the secret reference is then `my-project-my-app` in the secret namespace (Assuming `--convert-secret-name=false`). +The secrets are loaded based on how the environment variables `$ARGOCD_APP_NAME` and `$ARGOCD_APP_NAMESPACE` are used. If an application is in a project, the value of `$ARGOCD_APP_NAME` looks like this: `_`. For example, if the application `my-app` is in the project `my-project`, the value of `$ARGOCD_APP_NAME` is `my-project_my-app`. All special characters within are converted to `-` (dash). For example, if the application `my-app` is in the project `my-project`, the value of `$ARGOCD_APP_NAME` is `my-project-my-app`. So the secret reference is then `my-project-my-app` in the secret namespace (Assuming `--convert-secret-name=false`). -By default the `--convert-secret-name` is enabled. This removes the project prefix from the secret. If you create an application `test` in the namespace `test-reserved` the plugin is looking for private keys in the secret `test` in the namespace `test-reserved`. The is not considered in this approach which helps endusers to keep it simple. +By default the `--convert-secret-name` is enabled. This removes the project prefix from the secret. If you create an application `test` in the namespace `test-reserved` the plugin is looking for private keys in the secret `test` in the namespace `test-reserved`. -The values for the secret name and namespace can also be set constant, however this way you lose the multi-tenancy aspect of the secrets management: +The values for the secret name and namespace can also be set explicitly, however this way you lose the multi-tenancy aspect of the secrets management: ``` subst render --secret-name static-name --secret-namespace static-namespace . @@ -169,29 +167,16 @@ See below how to work with the different decryption providers. ### EJSON -[EJSON](https://github.com/Shopify/ejson) allows simple secrets management. I like it, because you can rencrypt secrets without having the private key, which is sometimes useful. - -You can encrypt entire files using EJSON. The file must be in json format (which is fun for kustomize). The entire file will be encrypted, which may not bes useful in all cases. - - -### SOPS - -[SOPS](https://github.com/mozilla/sops) is commonly known and also used by [FluxCD](https://fluxcd.io/flux/guides/mozilla-sops/). - +[EJSON](https://github.com/Shopify/ejson) allows simple secrets management. +You can encrypt entire files using EJSON. The file must be in json format. The entire file will be encrypted, which may not bes useful in all cases. ### Kubernetes For all decryptors you can create a kubernetes secret, which contains the private information for secret decryption. - - - - - # Running it - ## Local installation **Brew** @@ -213,10 +198,4 @@ https://github.com/bedag/subst/releases ## ArgoCD Plugin - - ## CI/CD - -TBD - - diff --git a/go.mod b/go.mod index fd53bad..44486a9 100644 --- a/go.mod +++ b/go.mod @@ -6,17 +6,19 @@ toolchain go1.23.0 require ( github.com/BurntSushi/toml v1.4.0 + github.com/KimMachineGun/automemlimit v0.6.1 github.com/MakeNowJust/heredoc v1.0.0 github.com/Masterminds/sprig/v3 v3.3.0 github.com/Shopify/ejson v1.5.2 + github.com/adberger/spruce v0.0.7 github.com/geofffranks/simpleyaml v0.0.0-20161109204137-c9320f076de5 - github.com/geofffranks/spruce v1.31.1 github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.14.0 github.com/starkandwayne/goutils v0.0.0-20190115202530-896b8a6904be github.com/stretchr/testify v1.9.0 + go.uber.org/automaxprocs v1.6.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/apimachinery v0.31.0 @@ -33,9 +35,13 @@ require ( github.com/Masterminds/semver/v3 v3.3.0 // indirect github.com/aws/aws-sdk-go v1.55.5 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cilium/ebpf v0.9.1 // indirect github.com/cloudfoundry-community/vaultkv v0.7.0 // indirect + github.com/containerd/cgroups/v3 v3.0.1 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/docker/go-units v0.4.0 // indirect github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -46,6 +52,7 @@ require ( github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect + github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect @@ -77,12 +84,15 @@ require ( github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/gomega v1.33.1 // indirect + github.com/opencontainers/runtime-spec v1.0.2 // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.7.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect @@ -94,7 +104,7 @@ require ( golang.org/x/crypto v0.27.0 // indirect golang.org/x/net v0.27.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sys v0.25.0 // indirect + golang.org/x/sys v0.26.0 // indirect golang.org/x/term v0.24.0 // indirect golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/go.sum b/go.sum index 39ece01..ab75b6d 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/KimMachineGun/automemlimit v0.6.1 h1:ILa9j1onAAMadBsyyUJv5cack8Y1WT26yLj/V+ulKp8= +github.com/KimMachineGun/automemlimit v0.6.1/go.mod h1:T7xYht7B8r6AG/AqFcUdc7fzd2bIdBKmepfP2S1svPY= github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= @@ -55,6 +57,8 @@ github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/Shopify/ejson v1.5.2 h1:sXUlmNd5MFHfxIvchQqkbksYmKmHb05coSYhMpWpUNs= github.com/Shopify/ejson v1.5.2/go.mod h1:bVvQ3MaBCfMOkIp1rWZcot3TruYXCc7qUUbI1tjs/YM= +github.com/adberger/spruce v0.0.7 h1:+JJgMhPpQc+gZz4X16tD54PPoO2HoUUmOQsAzU0L1gs= +github.com/adberger/spruce v0.0.7/go.mod h1:DBnNLJ0E4s1sPUzQR54/rniIx7MshmluK7jxdqZp+HU= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= @@ -63,13 +67,18 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.9.1 h1:64sn2K3UKw8NbP/blsixRpF3nXuyhz/VjRlRzvlBRu4= +github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudfoundry-community/vaultkv v0.7.0 h1:VFq0TQxGIxuJuqXKDlY73XneOQyKKTEBA8EwqKI3OOU= github.com/cloudfoundry-community/vaultkv v0.7.0/go.mod h1:D17jAL9n2GS66nbapOU7vRkGQ2D5zhsnyhCuspfNDlg= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/containerd/cgroups/v3 v3.0.1 h1:4hfGvu8rfGIwVIDd+nLzn/B9ZXx4BcCjzt5ToenJRaE= +github.com/containerd/cgroups/v3 v3.0.1/go.mod h1:/vtwk1VXrtoa5AaZLkypuOJgA/6DyPMZHJPGQNtlHnw= github.com/coreos/go-oidc/v3 v3.5.0/go.mod h1:ecXRtV4romGPeO6ieExAsUK9cb/3fp9hXNz1tlv8PIM= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -78,6 +87,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad h1:Qk76DOWdOp+GlyDKBAG3Klr9cn7N+LcYc82AZ2S7+cA= github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad/go.mod h1:mPKfmRa823oBIgl2r20LeMSpTAteW5j7FLkc0vjmzyQ= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= @@ -101,8 +112,6 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/geofffranks/simpleyaml v0.0.0-20161109204137-c9320f076de5 h1:5AjbNPs5ax5Rf1/FeG8tLUYOoEbEDvcMvBgBUH4fDRM= github.com/geofffranks/simpleyaml v0.0.0-20161109204137-c9320f076de5/go.mod h1:EoVmbOOR2VpnWfvsZ1wVdjvUbitLYk1SYxGTssyjW4s= -github.com/geofffranks/spruce v1.31.1 h1:U83FUWJBSe7zXrPt9uPLDH7pMvC4mEaieoxA7jcRqmI= -github.com/geofffranks/spruce v1.31.1/go.mod h1:48uMcwndjciQpBE00shdZ4mSGqFg2xjEL6ruQKR3HFY= github.com/geofffranks/yaml v0.0.0-20161117152608-9f2fe4b6f295 h1:CxigGHNaNtLTrnMveo9CjJjgXTuZpbLvuOvY/+c1v8g= github.com/geofffranks/yaml v0.0.0-20161117152608-9f2fe4b6f295/go.mod h1:+Qu4YOxbpR+Dn8JVzOTjJKWt3EZkEQD918wX+CkNcbE= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= @@ -126,6 +135,7 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8Wd github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -299,6 +309,10 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= @@ -310,6 +324,8 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -325,6 +341,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smarty/assertions v1.16.0 h1:EvHNkdRA4QHMrn75NZSoUQ/mAUXAYWfatfB01yTCzfY= github.com/smarty/assertions v1.16.0/go.mod h1:duaaFdCS0K9dnoM50iyek/eYINOZ64gbh1Xlf6LG7AI= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= @@ -382,6 +400,8 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.starlark.net v0.0.0-20221205180719-3fd0dac74452 h1:JZtNuL6LPB+scU5yaQ6hqRlJFRiddZm2FwRt2AQqtHA= go.starlark.net v0.0.0-20221205180719-3fd0dac74452/go.mod h1:kIVgS18CjmEC3PqMd5kaJSGEifyV/CeB9x506ZJ1Vbk= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -513,6 +533,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -556,8 +577,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/internal/utils/file.go b/internal/utils/file.go index 805b116..f2c918d 100644 --- a/internal/utils/file.go +++ b/internal/utils/file.go @@ -53,7 +53,8 @@ func (f *File) JSON() ([]byte, error) { if err != nil { return nil, err } - return json.Marshal(d) + //lint:ignore SA1026 This seems to be a false positive + return json.Marshal(d) //nolint:all } func (f *File) SPRUCE() (map[interface{}]interface{}, error) { diff --git a/internal/wrapper/spruce.go b/internal/wrapper/spruce.go index cac47a1..e901e29 100644 --- a/internal/wrapper/spruce.go +++ b/internal/wrapper/spruce.go @@ -3,7 +3,7 @@ package wrapper import ( "fmt" - "github.com/geofffranks/spruce" + "github.com/adberger/spruce" ) // Run Spruce Eval and return evaluator diff --git a/pkg/config/config.go b/pkg/config/config.go index fa48e49..58ba49f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -28,8 +28,6 @@ type Configuration struct { KubeAPI string `mapstructure:"kube-api"` Output string `mapstructure:"output"` ConvertSecretname bool `mapstructure:"convert-secret-name"` - SopSKeyring string `mapstructure:"sops-keyring"` - SopsTempKeyring bool `mapstructure:"sops-temp-keyring"` } func LoadConfiguration(cfgFile string, cmd *cobra.Command, directory string) (*Configuration, error) { diff --git a/pkg/subst/substitutions.go b/pkg/subst/substitutions.go index c8c8000..1e05d59 100644 --- a/pkg/subst/substitutions.go +++ b/pkg/subst/substitutions.go @@ -8,10 +8,10 @@ import ( "regexp" "text/template" + "github.com/adberger/spruce" decrypt "github.com/bedag/subst/internal/decryptors" "github.com/bedag/subst/internal/utils" "github.com/bedag/subst/internal/wrapper" - "github.com/geofffranks/spruce" "github.com/rs/zerolog/log" "sigs.k8s.io/kustomize/api/resmap" ) diff --git a/subst/cmd/render.go b/subst/cmd/render.go index fa4a2f3..6b86c23 100644 --- a/subst/cmd/render.go +++ b/subst/cmd/render.go @@ -50,10 +50,6 @@ func addRenderFlags(flags *flag.FlagSet) { flags.StringSlice("ejson-key", []string{}, heredoc.Doc(` Specify EJSON Private key used for decryption. May be specified multiple times or separate values with commas`)) - flags.String("sops-keyring", "", heredoc.Doc(` - Path to local GPG keyring`)) - flags.Bool("sops-temp-keyring", true, heredoc.Doc(` - Creates for each execution a dedicated keyring which is automatically deleted after execution. If false, uses the default keyring`)) flags.Bool("skip-decrypt", false, heredoc.Doc(` Skip decryption`)) flags.String("env-regex", "^ARGOCD_ENV_.*$", heredoc.Doc(` @@ -64,6 +60,8 @@ func addRenderFlags(flags *flag.FlagSet) { } func render(cmd *cobra.Command, args []string) error { + start := time.Now() // Start time measurement + dir, err := rootDirectory(args) if err != nil { return err @@ -83,7 +81,6 @@ func render(cmd *cobra.Command, args []string) error { return err } - start := time.Now() // Start time measurement if m != nil { err = m.Build() if err != nil { @@ -106,7 +103,7 @@ func render(cmd *cobra.Command, args []string) error { } } elapsed := time.Since(start) // Calculate elapsed time - log.Debug().Msgf("Build time: %s", elapsed) + log.Debug().Msgf("Build time for rendering: %s", elapsed) return nil } diff --git a/subst/cmd/root.go b/subst/cmd/root.go index 9d38f44..3abf577 100644 --- a/subst/cmd/root.go +++ b/subst/cmd/root.go @@ -2,19 +2,33 @@ package cmd import ( "fmt" + "log" + "log/slog" "os" "path/filepath" + "runtime/pprof" + "strconv" + "sync" + "github.com/KimMachineGun/automemlimit/memlimit" "github.com/rs/zerolog" flag "github.com/spf13/pflag" "github.com/MakeNowJust/heredoc" "github.com/spf13/cobra" + + "go.uber.org/automaxprocs/maxprocs" ) var ( - cfgFile string - v string + cfgFile string + v string + m float64 + p int + cpuProfile bool + memProfile bool + cpuProfileFile string + memProfileFile string ) func NewRootCmd() *cobra.Command { @@ -32,12 +46,30 @@ func NewRootCmd() *cobra.Command { if err := setUpLogs(v); err != nil { return err } + if err := setUpMemLimitRatio(m); err != nil { + return err + } + if err := setUpMaxProcs(p); err != nil { + return err + } + onStopProfiling = profilingInit() return nil } //Default value is the warn level cmd.PersistentFlags().StringVarP(&v, "verbosity", "v", zerolog.WarnLevel.String(), "Log level (debug, info, warn, error, fatal, panic") + //Default value is 0.1 (10%) + cmd.PersistentFlags().Float64VarP(&m, "memlimitratio", "m", 0.1, "Overwrite GOMEMLIMIT which the command can allocate (default: 0.1 which means 10%)") + //Default value is inferred from cgroups or system + cmd.PersistentFlags().IntVarP(&p, "maxprocs", "p", 0, "Overwrite GOMAXPROCS for the command to use (default: 0 which means respect cgroup or system)") + + cmd.PersistentFlags().BoolVar(&cpuProfile, "cpu-profile", false, "write cpu profile to file") + cmd.PersistentFlags().BoolVar(&memProfile, "mem-profile", false, "write memory profile to file") + + cmd.PersistentFlags().StringVar(&cpuProfileFile, "cpu-profile-file", "cpu.prof", "write cpu profile to file") + cmd.PersistentFlags().StringVar(&memProfileFile, "mem-profile-file", "mem.prof", "write memory profile to file") + cmd.AddCommand(newDiscoverCmd()) cmd.AddCommand(newVersionCmd()) cmd.AddCommand(newGenerateDocsCmd()) @@ -52,6 +84,7 @@ func NewRootCmd() *cobra.Command { // Execute runs the application func Execute() { + defer stopProfiling() if err := NewRootCmd().Execute(); err != nil { fmt.Println(err) os.Exit(1) @@ -68,6 +101,112 @@ func setUpLogs(level string) error { return nil } +// setUpMemLimitRatio set the memlimit ratio +func setUpMemLimitRatio(ratio float64) error { + _, err := memlimit.SetGoMemLimitWithOpts( + memlimit.WithRatio(ratio), + memlimit.WithProvider( + memlimit.ApplyFallback( + memlimit.FromCgroup, + memlimit.FromSystem, + ), + ), + memlimit.WithLogger(slog.Default()), + ) + if err != nil { + return err + } + return nil +} + +// setUpMaxProcs set the max procs +func setUpMaxProcs(procs int) error { + if procs > 0 { + os.Setenv("GOMAXPROCS", strconv.Itoa(procs)) + } + _, err := maxprocs.Set(maxprocs.Logger(log.Printf)) + if err != nil { + return err + } + return nil +} + +// profilingInit starts cpu and memory profiling if enabled. +// It returns a function to stop profiling. +func profilingInit() (stop func()) { + // doOnStop is a list of functions to be called on stop + var doOnStop []func() + + // stop calls all necessary functions to stop profiling + stop = func() { + for _, d := range doOnStop { + if d != nil { + d() + } + } + } + + // Start cpu profiling if enabled + if cpuProfile { + + fmt.Println("cpu profile enabled") + + // Create profiling file + f, err := os.Create(cpuProfileFile) + if err != nil { + fmt.Println("could not create cpu profile file") + return stop + } + + // Start profiling + err = pprof.StartCPUProfile(f) + if err != nil { + fmt.Println("could not start cpu profiling") + return stop + } + + // Add function to stop cpu profiling to doOnStop list + doOnStop = append(doOnStop, func() { + pprof.StopCPUProfile() + _ = f.Close() + fmt.Println("cpu profile stopped") + }) + } + + // Start memory profiling if enabled + if memProfile { + + fmt.Println("memory profile enabled") + + // Create profiling file + f, err := os.Create(memProfileFile) + if err != nil { + fmt.Println("could not create memory profile file") + return stop + } + + // Add function to stop memory profiling to doOnStop list + doOnStop = append(doOnStop, func() { + _ = pprof.WriteHeapProfile(f) + _ = f.Close() + fmt.Println("memory profile stopped") + }) + } + + return stop +} + +var onStopProfiling func() +var profilingOnce sync.Once + +// stopProfiling triggers _stopProfiling. +// It's safe to be called multiple times. +func stopProfiling() { + if onStopProfiling != nil { + profilingOnce.Do(onStopProfiling) + } +} + func addCommonFlags(flags *flag.FlagSet) { flags.StringVar(&cfgFile, "config", "", "Config file") flags.String("file-regex", "(subst\\.yaml|.*(ejson))", heredoc.Doc(` diff --git a/subst/cmd/root_test.go b/subst/cmd/root_test.go new file mode 100644 index 0000000..bead637 --- /dev/null +++ b/subst/cmd/root_test.go @@ -0,0 +1,22 @@ +package cmd + +import ( + "runtime/debug" + "testing" +) + +func BenchmarkExecute(b *testing.B) { + cmd := NewRootCmd() + cmd.SetArgs([]string{ + "render", + "--skip-decrypt", + "/Users/adrian/git/inventory/clusters/k8s-bedag-root-dev", + }) + for i := 0; i < b.N; i++ { + debug.SetGCPercent(800) + err := cmd.Execute() + if err != nil { + b.Errorf("Error: %v", err) + } + } +} diff --git a/subst/cmd/substitutions.go b/subst/cmd/substitutions.go index d559577..42352bb 100644 --- a/subst/cmd/substitutions.go +++ b/subst/cmd/substitutions.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "time" "github.com/MakeNowJust/heredoc" "github.com/bedag/subst/internal/utils" @@ -28,6 +29,8 @@ func newSubstitutionsCmd() *cobra.Command { } func substitutions(cmd *cobra.Command, args []string) error { + start := time.Now() // Start time measurement + dir, err := rootDirectory(args) if err != nil { return err @@ -62,6 +65,8 @@ func substitutions(cmd *cobra.Command, args []string) error { } } } + elapsed := time.Since(start) // Calculate elapsed time + log.Debug().Msgf("Build time for substitutions: %s", elapsed) return nil } diff --git a/subst/main.go b/subst/main.go index 1cce116..9e5d703 100644 --- a/subst/main.go +++ b/subst/main.go @@ -1,9 +1,12 @@ package main import ( + "runtime/debug" + "github.com/bedag/subst/subst/cmd" ) func main() { + debug.SetGCPercent(100) cmd.Execute() }