diff --git a/cmd/plax/main.go b/cmd/plax/main.go index 409e9e6..589a748 100644 --- a/cmd/plax/main.go +++ b/cmd/plax/main.go @@ -80,7 +80,7 @@ func main() { testSuiteName = flag.String("test-suite", "", "Name for JUnit test suite") logLevel = flag.String("log", "info", "log level (info, debug, none)") retry = flag.String("retry", "", `Specify retries: number or {"N":N,"Delay":"1s","DelayFactor":1.5}`) - redact = flag.Bool("redact", false, "Use redaction gear") + redact = flag.Bool("redact", true, "Use redaction gear") testRedactPattern = flag.String("check-redact-regexp", "", "regular expression to use for checking redactions (with no test executed)") testRedactString = flag.String("check-redact", "", "input string to use for -check-redact-regexp") diff --git a/cmd/plaxrun/main.go b/cmd/plaxrun/main.go index 7353b5f..0a2546e 100644 --- a/cmd/plaxrun/main.go +++ b/cmd/plaxrun/main.go @@ -58,19 +58,19 @@ func main() { var ( trps = &dsl.TestRunParams{ - Bindings: make(plaxDsl.Bindings), - IncludeDirs: dsl.IncludeDirList{wd}, - Filename: flag.String("run", "spec.yaml", "Filename for test run specification"), - Dir: flag.String("dir", ".", "Directory containing test files"), + Bindings: make(plaxDsl.Bindings), + IncludeDirs: dsl.IncludeDirList{wd}, + Filename: flag.String("run", "spec.yaml", "Filename for test run specification"), + Dir: flag.String("dir", ".", "Directory containing test files"), ReportPluginDir: flag.String("reportPluginDir", "plugins/report", "Directory containing the report plugins"), - EmitJSON: flag.Bool("json", false, "Emit JSON test output; instead of JUnit XML"), - Groups: dsl.TestGroupList{}, - Verbose: flag.Bool("v", true, "Verbosity"), - LogLevel: flag.String("log", "info", "Log level (info, debug, none)"), - Labels: flag.String("labels", "", "Labels for tests to run"), - SuiteName: flag.String("s", "", "Suite name to execute; -t options represent the tests in the suite to execute"), - Priority: flag.Int("priority", -1, "Test priority"), - Redact: flag.Bool("redact", false, "enable redactions when -log debug"), + EmitJSON: flag.Bool("json", false, "Emit JSON test output; instead of JUnit XML"), + Groups: dsl.TestGroupList{}, + Verbose: flag.Bool("v", true, "Verbosity"), + LogLevel: flag.String("log", "info", "Log level (info, debug, none)"), + Labels: flag.String("labels", "", "Labels for tests to run"), + SuiteName: flag.String("s", "", "Suite name to execute; -t options represent the tests in the suite to execute"), + Priority: flag.Int("priority", -1, "Test priority"), + Redact: flag.Bool("redact", true, "enable redactions when -log debug"), } vers = flag.Bool("version", false, "Print version and then exit") ) diff --git a/cmd/plaxrun/plugins/report/octane/go.sum b/cmd/plaxrun/plugins/report/octane/go.sum index 3bee979..9761f77 100644 --- a/cmd/plaxrun/plugins/report/octane/go.sum +++ b/cmd/plaxrun/plugins/report/octane/go.sum @@ -9,6 +9,20 @@ github.com/apex/log v1.0.0/go.mod h1:yA770aXIDQrhVOIGurT/pVdfCpSq1GQV/auzMN5fzvY github.com/avarabyeu/goRP/v5 v5.0.1/go.mod h1:6rg0WJHFysx5jdEdUk8NbFxkEKwVuypxeZNHPd9wnmg= github.com/aws/aws-sdk-go v1.15.0/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.40.4/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= +github.com/aws/aws-sdk-go-v2/config v1.18.15/go.mod h1:vS0tddZqpE8cD9CyW0/kITHF5Bq2QasW9Y1DFHD//O0= +github.com/aws/aws-sdk-go-v2/credentials v1.13.15/go.mod h1:vRMLMD3/rXU+o6j2MW5YefrGMBmdTvkLLGqFwMLBHQc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30/go.mod h1:vsbq62AOBwQ1LJ/GWKFxX8beUEYeRp/Agitrxee2/qM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44= +github.com/aws/aws-sdk-go-v2/service/kinesis v1.17.6/go.mod h1:CHlutPX3XBKs83BLG5WVROg9Ob97F7THaoIuTbPJVAg= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.4/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.5/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -53,8 +67,9 @@ github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaW github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= diff --git a/cmd/plaxrun/plugins/report/stdout/go.sum b/cmd/plaxrun/plugins/report/stdout/go.sum index 65616f8..9357d04 100644 --- a/cmd/plaxrun/plugins/report/stdout/go.sum +++ b/cmd/plaxrun/plugins/report/stdout/go.sum @@ -9,6 +9,20 @@ github.com/apex/log v1.0.0/go.mod h1:yA770aXIDQrhVOIGurT/pVdfCpSq1GQV/auzMN5fzvY github.com/avarabyeu/goRP/v5 v5.0.1/go.mod h1:6rg0WJHFysx5jdEdUk8NbFxkEKwVuypxeZNHPd9wnmg= github.com/aws/aws-sdk-go v1.15.0/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.40.4/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go-v2 v1.17.5/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10/go.mod h1:VeTZetY5KRJLuD/7fkQXMU6Mw7H5m/KP2J5Iy9osMno= +github.com/aws/aws-sdk-go-v2/config v1.18.15/go.mod h1:vS0tddZqpE8cD9CyW0/kITHF5Bq2QasW9Y1DFHD//O0= +github.com/aws/aws-sdk-go-v2/credentials v1.13.15/go.mod h1:vRMLMD3/rXU+o6j2MW5YefrGMBmdTvkLLGqFwMLBHQc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.23/go.mod h1:mOtmAg65GT1HIL/HT/PynwPbS+UG0BgCZ6vhkPqnxWo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.29/go.mod h1:Dip3sIGv485+xerzVv24emnjX5Sg88utCL8fwGmCeWg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.23/go.mod h1:mr6c4cHC+S/MMkrjtSlG4QA36kOznDep+0fga5L/fGQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.30/go.mod h1:vsbq62AOBwQ1LJ/GWKFxX8beUEYeRp/Agitrxee2/qM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.23/go.mod h1:9uPh+Hrz2Vn6oMnQYiUi/zbh3ovbnQk19YKINkQny44= +github.com/aws/aws-sdk-go-v2/service/kinesis v1.17.6/go.mod h1:CHlutPX3XBKs83BLG5WVROg9Ob97F7THaoIuTbPJVAg= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.4/go.mod h1:jtLIhd+V+lft6ktxpItycqHqiVXrPIRjWIsFIlzMriw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4/go.mod h1:zVwRrfdSmbRZWkUkWjOItY7SOalnFnq/Yg2LVPqDjwc= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.5/go.mod h1:1mKZHLLpDMHTNSYPJ7qrcnCQdHCWsNQaT0xRvq2u80s= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -52,8 +66,9 @@ github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaW github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= diff --git a/demos/redactions.yaml b/demos/redactions.yaml index bfb4f1f..5bb1dd6 100644 --- a/demos/redactions.yaml +++ b/demos/redactions.yaml @@ -47,7 +47,6 @@ spec: Use the previous regexp that has a group. Suppress output. payload: '"I love really thin pancakes."' - recv: - suppress: true pattern: '"?*x"' guard: | return 0 < bs['?*x'].indexOf("pancakes") diff --git a/doc/chan_kdspub.md b/doc/chan_kdspub.md new file mode 100644 index 0000000..ea59202 --- /dev/null +++ b/doc/chan_kdspub.md @@ -0,0 +1,12 @@ +## `kdspub` + +This channel consumes messages from a Kinesis stream. + +### Options + + +1. `StreamName` (string) is of course the name of the KDS. + +1. `BufferSize` (int) is the size of the underlying channel buffer. + Defaults to DefaultChanBufferSize. + diff --git a/doc/manual.md b/doc/manual.md index a32b548..272ff58 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -76,7 +76,7 @@ Usage of plax: -priority int Optional lowest priority (where larger numbers mean lower priority!); negative means all (default -1) -redact - Use redaction gear + Use redaction gear (default true) -retry string Specify retries: number or {"N":N,"Delay":"1s","DelayFactor":1.5} -seed int @@ -843,7 +843,9 @@ log redactions: 1. Values with binding names that start with `X_` (ignoring non-alphabetic prefix characters like `?`) will be redacted from - `debug` log output. + `debug` log output if `-redact=true` (default is true). + If the redacted `X_` values are needed for debugging, use `-redact=false` + locally. 1. In a test, Javascript (usually executed via a `run` step) can add redactions using the functions `redactRegexp` and `redactString`. diff --git a/doc/plaxrun.md b/doc/plaxrun.md index 2dc4305..d35d1fa 100644 --- a/doc/plaxrun.md +++ b/doc/plaxrun.md @@ -50,7 +50,7 @@ Usage of plaxrun: -priority int Test priority (default -1) -redact - enable redactions when -log debug + enable redactions when -log debug (default true) -run string Filename for test run specification (default "spec.yaml") -s string @@ -439,8 +439,10 @@ log redactions: 1. Values with binding names that start with `X_` (ignoring non-alphabetic prefix characters like `?`) will be redacted from - `debug` log output. - + `debug` log output if `-redact=true` (default is true). + If the redacted `X_` values are needed for debugging, use `-redact=false` + locally. + 1. In a test, Javascript (usually executed via a `run` step) can add redactions using the functions `redactRegexp` and `redactString`. See documentation above for usage information. diff --git a/dsl/ctx.go b/dsl/ctx.go index 6a27b7d..85c2745 100644 --- a/dsl/ctx.go +++ b/dsl/ctx.go @@ -116,8 +116,12 @@ func (c *Ctx) SetLogLevel(level string) error { } // bindingRedactions adds redaction patterns for values of binding -// variables that start with X_. +// variables that start with X_ if redact is true func (ctx *Ctx) BindingsRedactions(bs Bindings) error { + if !ctx.Redact { + return nil + } + for p, v := range bs { if WantsRedaction(p) { var s string diff --git a/dsl/spec.go b/dsl/spec.go index 6427764..2f1fffb 100644 --- a/dsl/spec.go +++ b/dsl/spec.go @@ -555,8 +555,6 @@ type Recv struct { // Max attempts to receive a message; optionally for a specific topic Attempts int `json:",omitempty" yaml:",omitempty` - Suppress bool `json:",omitempty" yaml:",omitempty` - ch Chan } @@ -636,7 +634,6 @@ func (r *Recv) Substitute(ctx *Ctx, t *Test) (*Recv, error) { Run: run, Schema: r.Schema, Attempts: r.Attempts, - Suppress: r.Suppress, ch: r.ch, }, nil } @@ -698,10 +695,8 @@ func (r *Recv) Exec(ctx *Ctx, t *Test) error { ctx.Indf(" Recv timeout (%v)", timeout) return fmt.Errorf("timeout after %s waiting for %s", timeout, r.Pattern) case m := <-in: - if !r.Suppress { - ctx.Indf(" Recv dequeuing topic '%s' (vs '%s')", m.Topic, r.Topic) - ctx.Inddf(" %s", m.Payload) - } + ctx.Indf(" Recv dequeuing topic '%s' (vs '%s')", m.Topic, r.Topic) + ctx.Inddf(" %s", m.Payload) var ( err error @@ -721,9 +716,7 @@ func (r *Recv) Exec(ctx *Ctx, t *Test) error { } bss, err = RegexpMatch(r.Regexp, m.Payload) } else { - if !r.Suppress { - ctx.Inddf(" pattern: %s", JSON(r.Pattern)) - } + ctx.Inddf(" pattern: %s", JSON(r.Pattern)) // target will be the target (message) for matching. var target interface{} @@ -746,9 +739,7 @@ func (r *Recv) Exec(ctx *Ctx, t *Test) error { return Brokenf("bad Recv Target: '%s'", r.Target) } - if !r.Suppress { - ctx.Inddf(" match target: %s", JSON(target)) - } + ctx.Inddf(" match target: %s", JSON(target)) if r.Schema != "" { if err := validateSchema(ctx, r.Schema, m.Payload); err != nil { @@ -763,9 +754,7 @@ func (r *Recv) Exec(ctx *Ctx, t *Test) error { return err } - if !r.Suppress { - ctx.Inddf(" bound pattern: %s", JSON(pattern)) - } + ctx.Inddf(" bound pattern: %s", JSON(pattern)) bss, err = match.Match(pattern, target, match.NewBindings()) } @@ -774,10 +763,6 @@ func (r *Recv) Exec(ctx *Ctx, t *Test) error { } ctx.Indf(" result: %v", 0 < len(bss)) - if !r.Suppress { - ctx.Inddf(" bss: %s", JSON(bss)) - } - if 0 < len(bss) { if 1 < len(bss) { @@ -867,6 +852,8 @@ func (r *Recv) Exec(ctx *Ctx, t *Test) error { } } + ctx.BindingsRedactions(t.Bindings) + ctx.Indf(" Recv satisfied") ctx.Inddf(" t.Bindings: %s", JSON(t.Bindings)) diff --git a/invoke/invoke.go b/invoke/invoke.go index 2b3d40c..96cdbfd 100644 --- a/invoke/invoke.go +++ b/invoke/invoke.go @@ -89,6 +89,11 @@ const ( func (inv *Invocation) Exec(ctx context.Context) (*junit.TestSuite, error) { dslCtx := dsl.NewCtx(ctx) dslCtx.Redact = inv.Redact + if inv.Redact { + //Add redactions for the X_ variables + dslCtx.Redactions.Add(".*{X_.*}=(.*)") + dslCtx.Redactions.Add(".*\"?X_.*\":(.*)}]") + } if len(inv.LogLevel) > 0 { if err := dslCtx.SetLogLevel(inv.LogLevel); err != nil {