Merge pull request #1 from draganm/scripting

Add Scripting

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.4 as builder
+FROM golang:1.17.6 as builder
 WORKDIR /app
 COPY go.mod .
 COPY go.sum .

README.md

@@ -5,10 +5,12 @@ Unlike many other templating mechanisms, this utility will never output an inval
 
 Under the hood, it parses the input yaml files, finds string values, interpolates environment values into them using bash-like interpolation syntax and then re-assembles the resulting YAML(s) that are printed on stdout.
 
+Also, there is an option of executing pre and post interpolation processors in ECMAScript 5.1(+).
+
 ## Use
 
 ```bash
-$ VAR=VALUE manifestor [<yaml file> ...] | kubectl apply -f -
+$ VAR=VALUE manifestor [--processors=] [<yaml file> ...] | kubectl apply -f -
 ```
 
 ## Interpolation format
@@ -62,3 +64,23 @@ spec:
                     servicePort: 3030
                   path: /api
 ```
+
+## Pre and post interpolation processors
+
+String interpolation of environment variables is quite limited.
+It won't cater for the cases where a number, boolean value or a whole sub-object needs to be changed, inserted or deleted.
+In order to cover those cases, we support executing so-called _processors_.
+
+A _processor_ is a JavaScript function that takes an object representing one entity from the manifest and can change it in place.
+
+### Naming convention
+
+TODO
+
+### Available functions
+
+TODO
+
+### Available objects
+
+TODO

go.mod

@@ -1,10 +1,19 @@
 module github.com/draganm/manifestor
 
-go 1.16
+go 1.17
 
 require (
+	github.com/dop251/goja v0.0.0-20220124171016-cfb079cdc7b4
 	github.com/drone/envsubst v1.0.2
-	github.com/pkg/errors v0.9.1
 	github.com/urfave/cli/v2 v2.3.0
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )
+
+require (
+	github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect
+	github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect
+	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
+	github.com/russross/blackfriday/v2 v2.0.1 // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	golang.org/x/text v0.3.6 // indirect
+)

go.sum

@@ -1,12 +1,24 @@
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 h1:Izz0+t1Z5nI16/II7vuEo/nHjodOg0p7+OiDpjX5t1E=
+github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/dop251/goja v0.0.0-20220124171016-cfb079cdc7b4 h1:gUXabLfCUjaNl7kLxGdaZaw1c5x33SGL9PEo6p/hfuo=
+github.com/dop251/goja v0.0.0-20220124171016-cfb079cdc7b4/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
+github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
 github.com/drone/envsubst v1.0.2 h1:dpYLMAspQHW0a8dZpLRKe9jCNvIGZPhCPrycZzIHdqo=
 github.com/drone/envsubst v1.0.2/go.mod h1:bkZbnc/2vh1M12Ecn7EYScpI4YGYU0etwLJICOWi8Z0=
+github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
+github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
@@ -15,8 +27,14 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5I
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -2,37 +2,107 @@ package main
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"os"
+	"strings"
 
+	"github.com/dop251/goja"
 	"github.com/drone/envsubst"
-	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v3"
 )
 
 func main() {
 	app := &cli.App{
+		Flags: []cli.Flag{
+			&cli.StringSliceFlag{
+				Name:    "processor",
+				EnvVars: []string{"PROCESSORS"},
+			},
+		},
 		Action: func(c *cli.Context) (err error) {
 			defer func() {
 				if err != nil {
 					err = cli.NewExitError(err.Error(), 1)
 				}
 			}()
 
+			env := map[string]string{}
+
+			for _, e := range os.Environ() {
+
+				kv := strings.SplitN(e, "=", 2)
+				if len(kv) != 2 {
+					return fmt.Errorf("while splitting env %q - got %d parts", e, len(kv))
+				}
+
+				env[kv[0]] = kv[1]
+			}
+
+			vm := goja.New()
+
+			for _, p := range c.StringSlice("processor") {
+				script, err := os.ReadFile(p)
+				if err != nil {
+					return fmt.Errorf("while reading script %s: %w", p, err)
+				}
+				vm.RunScript(p, string(script))
+			}
+
+			vm.Set("env", env)
+
+			vm.Set("parseYaml", func(docString string) (interface{}, error) {
+				var v interface{}
+				err := yaml.NewDecoder(strings.NewReader(docString)).Decode(&v)
+				if err != nil {
+					return nil, err
+				}
+				return v, nil
+			})
+
+			var preProcessors []func(interface{}) error
+			var postProcessors []func(interface{}) error
+
+			for _, k := range vm.GlobalObject().Keys() {
+
+				if strings.HasPrefix(k, "pre") {
+					v := vm.GlobalObject().Get(k)
+
+					var fn func(interface{}) error
+					err = vm.ExportTo(v, &fn)
+					if err != nil {
+						continue
+					}
+					preProcessors = append(preProcessors, wrap(k, fn))
+				}
+
+				if strings.HasPrefix(k, "post") {
+					v := vm.GlobalObject().Get(k)
+					var fn func(interface{}) error
+					err = vm.ExportTo(v, &fn)
+					if err != nil {
+						fmt.Println("err", err)
+						continue
+					}
+					postProcessors = append(postProcessors, wrap(k, fn))
+				}
+
+			}
+
 			enc := yaml.NewEncoder(os.Stdout)
 
 			for _, f := range c.Args().Slice() {
 				var b []byte
 				if f == "-" {
 					b, err = io.ReadAll(os.Stdin)
 					if err != nil {
-						return errors.Wrap(err, "while reading from stdin")
+						return fmt.Errorf("while reading from stdin: %w", err)
 					}
 				} else {
 					b, err = os.ReadFile(f)
 					if err != nil {
-						return errors.Wrapf(err, "while reading file %s", f)
+						return fmt.Errorf("while reading file %s: %w", f, err)
 					}
 
 				}
@@ -47,17 +117,31 @@ func main() {
 					}
 
 					if err != nil {
-						return errors.Wrapf(err, "while decoding yaml file %s", f)
+						return fmt.Errorf("while decoding yaml file %s: %w", f, err)
+					}
+
+					for _, p := range preProcessors {
+						err = p(obj)
+						if err != nil {
+							return fmt.Errorf("while pre processing %s: %w", f, err)
+						}
 					}
 
 					iobj, err := interpolate(obj)
 					if err != nil {
-						return errors.Wrapf(err, "while interpolating values into %s", f)
+						return fmt.Errorf("while interpolating values into %s: %w", f, err)
+					}
+
+					for _, p := range postProcessors {
+						err = p(iobj)
+						if err != nil {
+							return fmt.Errorf("while post processing %s: %w", f, err)
+						}
 					}
 
 					err = enc.Encode(iobj)
 					if err != nil {
-						return errors.Wrap(err, "while encoding interpolated manifests")
+						return fmt.Errorf("while encoding interpolated manifests: %w", err)
 					}
 
 				}
@@ -77,7 +161,7 @@ func interpolate(o interface{}) (interface{}, error) {
 		for mk, mv := range v {
 			nv, err := interpolate(mv)
 			if err != nil {
-				return nil, errors.Wrapf(err, "while interpolating map value for key %q", mk)
+				return nil, fmt.Errorf("while interpolating map value for key %q: %w", mk, err)
 			}
 			nm[mk] = nv
 		}
@@ -87,7 +171,7 @@ func interpolate(o interface{}) (interface{}, error) {
 		for i, sv := range v {
 			nv, err := interpolate(sv)
 			if err != nil {
-				return nil, errors.Wrapf(err, "while interpolating slice value with index %d", i)
+				return nil, fmt.Errorf("while interpolating slice value with index %d: %w", i, err)
 			}
 			ns[i] = nv
 		}

wrapped_function.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/dop251/goja"
+)
+
+func wrap(name string, f func(interface{}) error) func(interface{}) error {
+	return func(i interface{}) (err error) {
+		defer func() {
+			p := recover()
+			if p == nil {
+				return
+			}
+
+			ex, isException := p.(*goja.Exception)
+
+			if isException {
+				err = fmt.Errorf("while executing %s():\n%v", name, ex.String())
+			} else {
+				err = fmt.Errorf("while executing %s: %v", name, p)
+			}
+
+		}()
+
+		return f(i)
+	}
+}