From e866ce2452a72daf3c44d378f3e236d0716660cd Mon Sep 17 00:00:00 2001 From: robot-piglet Date: Tue, 31 Oct 2023 15:59:37 +0300 Subject: [PATCH 1/6] Intermediate changes --- .mapping.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.mapping.json b/.mapping.json index c244fcbae..07f6fe6d4 100644 --- a/.mapping.json +++ b/.mapping.json @@ -256,8 +256,8 @@ "examples/debug_and_profiling.yaml":"load/projects/pandora/examples/debug_and_profiling.yaml", "examples/http.jsonline":"load/projects/pandora/examples/http.jsonline", "examples/http.yaml":"load/projects/pandora/examples/http.yaml", - "go.mod":"gomod/go.mod", - "go.sum":"gomod/go.sum", + "go.mod":"load/projects/pandora/gomod/go.mod", + "go.sum":"load/projects/pandora/gomod/go.sum", "gomod/go.mod":"load/projects/pandora/gomod/go.mod", "gomod/go.sum":"load/projects/pandora/gomod/go.sum", "labels.json":"load/projects/pandora/labels.json", From 730083042a2f7ae978997ba9a450dd56bbfa09f2 Mon Sep 17 00:00:00 2001 From: sabevzenko Date: Tue, 31 Oct 2023 17:02:23 +0300 Subject: [PATCH 2/6] rus docs link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2a517a288..a172a0ee9 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Or use Pandora with [Yandex.Tank](https://yandextank.readthedocs.io/en/latest/co [Overload](https://overload.yandex.net). ### Documentation -[Documentation](documentation/eng/index.md) +[Documentation](docs/index.md) ### Old Documentation [ReadTheDocs](https://yandexpandora.readthedocs.io/) From 6702ac1e02c251ff79abccba0b1b04b00d0d2a0f Mon Sep 17 00:00:00 2001 From: sabevzenko Date: Wed, 1 Nov 2023 08:29:55 +0300 Subject: [PATCH 3/6] rus docs link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a172a0ee9..199239f6e 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Or use Pandora with [Yandex.Tank](https://yandextank.readthedocs.io/en/latest/co [Overload](https://overload.yandex.net). ### Documentation -[Documentation](docs/index.md) +[Documentation](https://yandex.github.io/pandora/) ### Old Documentation [ReadTheDocs](https://yandexpandora.readthedocs.io/) From afd5191c1f9dbe7206492abea111fb10f901c165 Mon Sep 17 00:00:00 2001 From: Guspan Tanadi <36249910+guspan-tanadi@users.noreply.github.com> Date: Thu, 2 Nov 2023 18:03:34 +0300 Subject: [PATCH 4/6] docs: bit misspells Pull Request resolved: https://github.com/yandex/pandora/pull/171 --- docs/eng/config.md | 1 - docs/eng/http-generator.md | 2 +- docs/eng/scenario-http-generator.md | 2 +- docs/rus/config.md | 1 - docs/rus/http-generator.md | 2 +- 5 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/eng/config.md b/docs/eng/config.md index 6855846c4..87f8e2e08 100644 --- a/docs/eng/config.md +++ b/docs/eng/config.md @@ -7,7 +7,6 @@ - [Basic configuration](#basic-configuration) - [Monitoring and Logging](#monitoring-and-logging) - [Variables from env and files](#variables-from-env-and-files) -- [Variables from env and files](#variables-from-env-and-files) ## Basic configuration diff --git a/docs/eng/http-generator.md b/docs/eng/http-generator.md index 43d3ad7b4..bd73b2a09 100644 --- a/docs/eng/http-generator.md +++ b/docs/eng/http-generator.md @@ -35,7 +35,7 @@ gun: uri-elements: 2 # URI elements used to autotagging. Default: 2 no-tag-only: true # When true, autotagged only ammo that has no tag before. Default: true httptrace: - dump: true # calculate responce bytes + dump: true # calculate response bytes trace: true # calculate different request stages: connect time, send time, latency, request bytes ``` diff --git a/docs/eng/scenario-http-generator.md b/docs/eng/scenario-http-generator.md index a1d24b35e..4455ca5f6 100644 --- a/docs/eng/scenario-http-generator.md +++ b/docs/eng/scenario-http-generator.md @@ -195,7 +195,7 @@ scenarios: ### Templater -The `uri`, `headers`, `body` fields are templateized. +The `uri`, `headers`, `body` fields are templatized. The standard go template is used. diff --git a/docs/rus/config.md b/docs/rus/config.md index 9e385e842..69364f17d 100644 --- a/docs/rus/config.md +++ b/docs/rus/config.md @@ -7,7 +7,6 @@ - [Basic configuration](#basic-configuration) - [Monitoring and Logging](#monitoring-and-logging) - [Variables from env and files](#variables-from-env-and-files) -- [Variables from env and files](#variables-from-env-and-files) ## Basic configuration diff --git a/docs/rus/http-generator.md b/docs/rus/http-generator.md index 31ccba8c0..80aedbd52 100644 --- a/docs/rus/http-generator.md +++ b/docs/rus/http-generator.md @@ -35,7 +35,7 @@ gun: uri-elements: 2 # URI elements used to autotagging. Default: 2 no-tag-only: true # When true, autotagged only ammo that has no tag before. Default: true httptrace: - dump: true # calculate responce bytes + dump: true # calculate response bytes trace: true # calculate different request stages: connect time, send time, latency, request bytes ``` From 61a99eee935eaad9e5ca4d0dd93e8e992c09e609 Mon Sep 17 00:00:00 2001 From: Nikolay Tuzov Date: Thu, 2 Nov 2023 18:04:07 +0300 Subject: [PATCH 5/6] added some tests and docs Added tests for: - monitoring/counter Also added a little docs for monitoring/counter Pull Request resolved: https://github.com/yandex/pandora/pull/131 --- lib/monitoring/counter.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/monitoring/counter.go b/lib/monitoring/counter.go index 17d4e8972..6c8744014 100644 --- a/lib/monitoring/counter.go +++ b/lib/monitoring/counter.go @@ -24,14 +24,17 @@ func (c *Counter) String() string { return strconv.FormatInt(c.i.Load(), 10) } +// Add adds given delta to a counter value func (c *Counter) Add(delta int64) { c.i.Add(delta) } +// Set sets given value as counter value func (c *Counter) Set(value int64) { c.i.Store(value) } +// Get returns counter value func (c *Counter) Get() int64 { return c.i.Load() } From f49499883b592820c95f1533a5f3909b521426b4 Mon Sep 17 00:00:00 2001 From: sabevzenko Date: Tue, 7 Nov 2023 17:22:03 +0300 Subject: [PATCH 6/6] std json-decoder with array payload std json-decoder with array payload std json-decoder --- .mapping.json | 3 - cli/cli.go | 2 +- components/providers/http/decoders/decoder.go | 2 +- .../providers/http/decoders/jsonline.go | 140 ++++- .../providers/http/decoders/jsonline/data.go | 42 -- .../http/decoders/jsonline/data_ffjson.go | 485 ------------------ .../http/decoders/jsonline/data_test.go | 61 --- .../providers/http/decoders/jsonline_test.go | 279 +++++++++- components/providers/http/provider_test.go | 51 +- go.mod | 1 - go.sum | 2 - 11 files changed, 393 insertions(+), 675 deletions(-) delete mode 100644 components/providers/http/decoders/jsonline/data.go delete mode 100644 components/providers/http/decoders/jsonline/data_ffjson.go delete mode 100644 components/providers/http/decoders/jsonline/data_test.go diff --git a/.mapping.json b/.mapping.json index 07f6fe6d4..d1f42f783 100644 --- a/.mapping.json +++ b/.mapping.json @@ -53,9 +53,6 @@ "components/providers/http/decoders/ammo/raw_ammo.go":"load/projects/pandora/components/providers/http/decoders/ammo/raw_ammo.go", "components/providers/http/decoders/decoder.go":"load/projects/pandora/components/providers/http/decoders/decoder.go", "components/providers/http/decoders/jsonline.go":"load/projects/pandora/components/providers/http/decoders/jsonline.go", - "components/providers/http/decoders/jsonline/data.go":"load/projects/pandora/components/providers/http/decoders/jsonline/data.go", - "components/providers/http/decoders/jsonline/data_ffjson.go":"load/projects/pandora/components/providers/http/decoders/jsonline/data_ffjson.go", - "components/providers/http/decoders/jsonline/data_test.go":"load/projects/pandora/components/providers/http/decoders/jsonline/data_test.go", "components/providers/http/decoders/jsonline_test.go":"load/projects/pandora/components/providers/http/decoders/jsonline_test.go", "components/providers/http/decoders/mock_decoder.go":"load/projects/pandora/components/providers/http/decoders/mock_decoder.go", "components/providers/http/decoders/raw.go":"load/projects/pandora/components/providers/http/decoders/raw.go", diff --git a/cli/cli.go b/cli/cli.go index f95462ce2..796a7dc59 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -24,7 +24,7 @@ import ( "go.uber.org/zap/zapcore" ) -const Version = "0.5.14" +const Version = "0.5.15" const defaultConfigFile = "load" const stdinConfigSelector = "-" diff --git a/components/providers/http/decoders/decoder.go b/components/providers/http/decoders/decoder.go index 193a52a31..dfe58b4be 100644 --- a/components/providers/http/decoders/decoder.go +++ b/components/providers/http/decoders/decoder.go @@ -71,7 +71,7 @@ func NewDecoder(conf config.Config, file io.ReadSeeker) (d Decoder, err error) { switch conf.Decoder { case config.DecoderJSONLine: - d = newJsonlineDecoder(file, conf, decodedConfigHeaders) + d, err = newJsonlineDecoder(file, conf, decodedConfigHeaders) case config.DecoderRaw: d = newRawDecoder(file, conf, decodedConfigHeaders) case config.DecoderURI: diff --git a/components/providers/http/decoders/jsonline.go b/components/providers/http/decoders/jsonline.go index 07d564c27..56e267599 100644 --- a/components/providers/http/decoders/jsonline.go +++ b/components/providers/http/decoders/jsonline.go @@ -3,25 +3,32 @@ package decoders import ( "bufio" "context" + "encoding/json" + "errors" + "fmt" "io" "net/http" - "strings" "sync" "github.com/yandex/pandora/components/providers/http/config" "github.com/yandex/pandora/components/providers/http/decoders/ammo" - "github.com/yandex/pandora/components/providers/http/decoders/jsonline" "github.com/yandex/pandora/core" "golang.org/x/xerrors" ) -func newJsonlineDecoder(file io.ReadSeeker, cfg config.Config, decodedConfigHeaders http.Header) *jsonlineDecoder { +func newJsonlineDecoder(file io.ReadSeeker, cfg config.Config, decodedConfigHeaders http.Header) (*jsonlineDecoder, error) { scanner := bufio.NewScanner(file) if cfg.MaxAmmoSize != 0 { var buffer []byte scanner.Buffer(buffer, cfg.MaxAmmoSize) } - return &jsonlineDecoder{ + + isArray, err := isArray(file) + if err != nil { + return nil, err + } + + decoder := &jsonlineDecoder{ protoDecoder: protoDecoder{ file: file, config: cfg, @@ -29,7 +36,30 @@ func newJsonlineDecoder(file io.ReadSeeker, cfg config.Config, decodedConfigHead }, scanner: scanner, pool: &sync.Pool{New: func() any { return &ammo.Ammo{} }}, + decoder: json.NewDecoder(file), } + if isArray { + ammos, err := decoder.readArray() + if err != nil { + return decoder, fmt.Errorf("cant read json array: %w", err) + } + decoder.ammos = ammos + } + return decoder, nil +} + +func isArray(r io.ReadSeeker) (bool, error) { + d := json.NewDecoder(r) + t, err := d.Token() + if err != nil { + return false, err + } + delim, ok := t.(json.Delim) + if !ok { + return false, errors.New("invalid json token") + } + _, err = r.Seek(0, io.SeekStart) + return delim.String() == "[", err } type jsonlineDecoder struct { @@ -37,6 +67,8 @@ type jsonlineDecoder struct { scanner *bufio.Scanner line uint pool *sync.Pool + decoder *json.Decoder + ammos []DecodedAmmo } func (d *jsonlineDecoder) Release(a core.Ammo) { @@ -50,36 +82,57 @@ func (d *jsonlineDecoder) LoadAmmo(ctx context.Context) ([]DecodedAmmo, error) { return d.protoDecoder.LoadAmmo(ctx, d.Scan) } +type entity struct { + // Host defines Host header to send. + // Request endpoint is defied by gun config. + Host string `json:"host"` + Method string `json:"method"` + URI string `json:"uri"` + // Headers defines headers to send. + // NOTE: Host header will be silently ignored. + Headers map[string]string `json:"headers"` + Tag string `json:"tag"` + // Body should be string, doublequotes should be escaped for json body + Body string `json:"body"` +} + func (d *jsonlineDecoder) Scan(ctx context.Context) (DecodedAmmo, error) { if d.config.Limit != 0 && d.ammoNum >= d.config.Limit { return nil, ErrAmmoLimit } + if d.ammos != nil { + return d.scanAmmos() + } for { if d.config.Passes != 0 && d.passNum >= d.config.Passes { return nil, ErrPassLimit } - - for d.scanner.Scan() { - d.line++ - data := d.scanner.Bytes() - if len(strings.TrimSpace(string(data))) == 0 { - continue + var da entity + err := d.decoder.Decode(&da) + if err != nil { + if err != io.EOF { + return nil, xerrors.Errorf("failed to decode ammo at line: %v; with err: %w", d.line+1, err) } + // go to next pass + } else { + d.line++ d.ammoNum++ - method, url, header, tag, body, err := jsonline.DecodeAmmo(data, d.decodedConfigHeaders) - if err != nil { - if !d.config.ContinueOnError { - return nil, xerrors.Errorf("failed to decode ammo at line: %v; data: %q, with err: %w", d.line+1, data, err) - } - // TODO: add log message about error - continue // skipping ammo + + header := d.decodedConfigHeaders.Clone() + for k, v := range da.Headers { + header.Set(k, v) + } + url := "http://" + da.Host + da.URI // schema will be rewrite in gun + var body []byte + if da.Body != "" { + body = []byte(da.Body) } a := d.pool.Get().(*ammo.Ammo) - err = a.Setup(method, url, body, header, tag) + err = a.Setup(da.Method, url, body, header, da.Tag) return a, err } - err := d.scanner.Err() + err = d.scanner.Err() if err != nil { return nil, err } @@ -93,10 +146,51 @@ func (d *jsonlineDecoder) Scan(ctx context.Context) (DecodedAmmo, error) { if err != nil { return nil, err } - d.scanner = bufio.NewScanner(d.file) - if d.config.MaxAmmoSize != 0 { - var buffer []byte - d.scanner.Buffer(buffer, d.config.MaxAmmoSize) + d.decoder = json.NewDecoder(d.file) + } +} + +func (d *jsonlineDecoder) readArray() ([]DecodedAmmo, error) { + var data []entity + err := d.decoder.Decode(&data) + if err != nil { + return nil, fmt.Errorf("cant readArray, err: %w", err) + } + result := make([]DecodedAmmo, len(data)) + for i, datum := range data { + header := d.decodedConfigHeaders.Clone() + for k, v := range datum.Headers { + header.Set(k, v) + } + url := "http://" + datum.Host + datum.URI // schema will be rewrite in gun + var body []byte + if datum.Body != "" { + body = []byte(datum.Body) + } + a := d.pool.Get().(*ammo.Ammo) + err = a.Setup(datum.Method, url, body, header, datum.Tag) + if err != nil { + return nil, fmt.Errorf("cant readArray, err: %w", err) } + result[i] = a + } + + return result, nil +} + +func (d *jsonlineDecoder) scanAmmos() (DecodedAmmo, error) { + length := len(d.ammos) + if length == 0 { + return nil, ErrNoAmmo + } + if d.config.Passes != 0 && d.passNum >= d.config.Passes { + return nil, ErrPassLimit + } + i := int(d.ammoNum) % length + a := d.ammos[i] + if d.ammoNum > 0 && i == length-1 { + d.passNum++ } + d.ammoNum++ + return a, nil } diff --git a/components/providers/http/decoders/jsonline/data.go b/components/providers/http/decoders/jsonline/data.go deleted file mode 100644 index 2f0122864..000000000 --- a/components/providers/http/decoders/jsonline/data.go +++ /dev/null @@ -1,42 +0,0 @@ -//go:generate github.com/pquerna/ffjson@latest data_ffjson.go - -package jsonline - -import ( - "net/http" - - "github.com/pkg/errors" -) - -// ffjson: noencoder -type data struct { - // Host defines Host header to send. - // Request endpoint is defied by gun config. - Host string `json:"host"` - Method string `json:"method"` - URI string `json:"uri"` - // Headers defines headers to send. - // NOTE: Host header will be silently ignored. - Headers map[string]string `json:"headers"` - Tag string `json:"tag"` - // Body should be string, doublequotes should be escaped for json body - Body string `json:"body"` -} - -func DecodeAmmo(jsonDoc []byte, baseHeader http.Header) (method string, url string, header http.Header, tag string, body []byte, err error) { - var d = new(data) - if err := d.UnmarshalJSON(jsonDoc); err != nil { - err = errors.WithStack(err) - return "", "", nil, "", nil, err - } - - header = baseHeader.Clone() - for k, v := range d.Headers { - header.Set(k, v) - } - url = "http://" + d.Host + d.URI - if d.Body != "" { - body = []byte(d.Body) - } - return d.Method, url, header, d.Tag, body, nil -} diff --git a/components/providers/http/decoders/jsonline/data_ffjson.go b/components/providers/http/decoders/jsonline/data_ffjson.go deleted file mode 100644 index 31ce2e0e9..000000000 --- a/components/providers/http/decoders/jsonline/data_ffjson.go +++ /dev/null @@ -1,485 +0,0 @@ -// Code generated by ffjson . DO NOT EDIT. -// source: data.go - -package jsonline - -import ( - "bytes" - "fmt" - fflib "github.com/pquerna/ffjson/fflib/v1" -) - -const ( - ffjtdatabase = iota - ffjtdatanosuchkey - - ffjtdataHost - - ffjtdataMethod - - ffjtdataURI - - ffjtdataHeaders - - ffjtdataTag - - ffjtdataBody -) - -var ffjKeydataHost = []byte("host") - -var ffjKeydataMethod = []byte("method") - -var ffjKeydataURI = []byte("uri") - -var ffjKeydataHeaders = []byte("headers") - -var ffjKeydataTag = []byte("tag") - -var ffjKeydataBody = []byte("body") - -// UnmarshalJSON umarshall json - template of ffjson -func (j *data) UnmarshalJSON(input []byte) error { - fs := fflib.NewFFLexer(input) - return j.UnmarshalJSONFFLexer(fs, fflib.FFParse_map_start) -} - -// UnmarshalJSONFFLexer fast json unmarshall - template ffjson -func (j *data) UnmarshalJSONFFLexer(fs *fflib.FFLexer, state fflib.FFParseState) error { - var err error - currentKey := ffjtdatabase - _ = currentKey - tok := fflib.FFTok_init - wantedTok := fflib.FFTok_init - -mainparse: - for { - tok = fs.Scan() - // println(fmt.Sprintf("debug: tok: %v state: %v", tok, state)) - if tok == fflib.FFTok_error { - goto tokerror - } - - switch state { - - case fflib.FFParse_map_start: - if tok != fflib.FFTok_left_bracket { - wantedTok = fflib.FFTok_left_bracket - goto wrongtokenerror - } - state = fflib.FFParse_want_key - continue - - case fflib.FFParse_after_value: - if tok == fflib.FFTok_comma { - state = fflib.FFParse_want_key - } else if tok == fflib.FFTok_right_bracket { - goto done - } else { - wantedTok = fflib.FFTok_comma - goto wrongtokenerror - } - - case fflib.FFParse_want_key: - // json {} ended. goto exit. woo. - if tok == fflib.FFTok_right_bracket { - goto done - } - if tok != fflib.FFTok_string { - wantedTok = fflib.FFTok_string - goto wrongtokenerror - } - - kn := fs.Output.Bytes() - if len(kn) <= 0 { - // "" case. hrm. - currentKey = ffjtdatanosuchkey - state = fflib.FFParse_want_colon - goto mainparse - } else { - switch kn[0] { - - case 'b': - - if bytes.Equal(ffjKeydataBody, kn) { - currentKey = ffjtdataBody - state = fflib.FFParse_want_colon - goto mainparse - } - - case 'h': - - if bytes.Equal(ffjKeydataHost, kn) { - currentKey = ffjtdataHost - state = fflib.FFParse_want_colon - goto mainparse - - } else if bytes.Equal(ffjKeydataHeaders, kn) { - currentKey = ffjtdataHeaders - state = fflib.FFParse_want_colon - goto mainparse - } - - case 'm': - - if bytes.Equal(ffjKeydataMethod, kn) { - currentKey = ffjtdataMethod - state = fflib.FFParse_want_colon - goto mainparse - } - - case 't': - - if bytes.Equal(ffjKeydataTag, kn) { - currentKey = ffjtdataTag - state = fflib.FFParse_want_colon - goto mainparse - } - - case 'u': - - if bytes.Equal(ffjKeydataURI, kn) { - currentKey = ffjtdataURI - state = fflib.FFParse_want_colon - goto mainparse - } - - } - - if fflib.SimpleLetterEqualFold(ffjKeydataBody, kn) { - currentKey = ffjtdataBody - state = fflib.FFParse_want_colon - goto mainparse - } - - if fflib.SimpleLetterEqualFold(ffjKeydataTag, kn) { - currentKey = ffjtdataTag - state = fflib.FFParse_want_colon - goto mainparse - } - - if fflib.EqualFoldRight(ffjKeydataHeaders, kn) { - currentKey = ffjtdataHeaders - state = fflib.FFParse_want_colon - goto mainparse - } - - if fflib.SimpleLetterEqualFold(ffjKeydataURI, kn) { - currentKey = ffjtdataURI - state = fflib.FFParse_want_colon - goto mainparse - } - - if fflib.SimpleLetterEqualFold(ffjKeydataMethod, kn) { - currentKey = ffjtdataMethod - state = fflib.FFParse_want_colon - goto mainparse - } - - if fflib.EqualFoldRight(ffjKeydataHost, kn) { - currentKey = ffjtdataHost - state = fflib.FFParse_want_colon - goto mainparse - } - - currentKey = ffjtdatanosuchkey - state = fflib.FFParse_want_colon - goto mainparse - } - - case fflib.FFParse_want_colon: - if tok != fflib.FFTok_colon { - wantedTok = fflib.FFTok_colon - goto wrongtokenerror - } - state = fflib.FFParse_want_value - continue - case fflib.FFParse_want_value: - - if tok == fflib.FFTok_left_brace || tok == fflib.FFTok_left_bracket || tok == fflib.FFTok_integer || tok == fflib.FFTok_double || tok == fflib.FFTok_string || tok == fflib.FFTok_bool || tok == fflib.FFTok_null { - switch currentKey { - - case ffjtdataHost: - goto handle_Host - - case ffjtdataMethod: - goto handle_Method - - case ffjtdataURI: - goto handle_URI - - case ffjtdataHeaders: - goto handle_Headers - - case ffjtdataTag: - goto handle_Tag - - case ffjtdataBody: - goto handle_Body - - case ffjtdatanosuchkey: - err = fs.SkipField(tok) - if err != nil { - return fs.WrapErr(err) - } - state = fflib.FFParse_after_value - goto mainparse - } - } else { - goto wantedvalue - } - } - } - -handle_Host: - - /* handler: j.Host type=string kind=string quoted=false*/ - - { - - { - if tok != fflib.FFTok_string && tok != fflib.FFTok_null { - return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for string", tok)) - } - } - - if tok == fflib.FFTok_null { - - } else { - - outBuf := fs.Output.Bytes() - - j.Host = string(string(outBuf)) - - } - } - - state = fflib.FFParse_after_value - goto mainparse - -handle_Method: - - /* handler: j.Method type=string kind=string quoted=false*/ - - { - - { - if tok != fflib.FFTok_string && tok != fflib.FFTok_null { - return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for string", tok)) - } - } - - if tok == fflib.FFTok_null { - - } else { - - outBuf := fs.Output.Bytes() - - j.Method = string(string(outBuf)) - - } - } - - state = fflib.FFParse_after_value - goto mainparse - -handle_URI: - - /* handler: j.URI type=string kind=string quoted=false*/ - - { - - { - if tok != fflib.FFTok_string && tok != fflib.FFTok_null { - return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for string", tok)) - } - } - - if tok == fflib.FFTok_null { - - } else { - - outBuf := fs.Output.Bytes() - - j.URI = string(string(outBuf)) - - } - } - - state = fflib.FFParse_after_value - goto mainparse - -handle_Headers: - - /* handler: j.Headers type=map[string]string kind=map quoted=false*/ - - { - - { - if tok != fflib.FFTok_left_bracket && tok != fflib.FFTok_null { - return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for ", tok)) - } - } - - if tok == fflib.FFTok_null { - j.Headers = nil - } else { - - j.Headers = make(map[string]string, 0) - - wantVal := true - - for { - - var k string - - var tmpJHeaders string - - tok = fs.Scan() - if tok == fflib.FFTok_error { - goto tokerror - } - if tok == fflib.FFTok_right_bracket { - break - } - - if tok == fflib.FFTok_comma { - if wantVal == true { - // TODO(pquerna): this isn't an ideal error message, this handles - // things like [,,,] as an array value. - return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok)) - } - continue - } else { - wantVal = true - } - - /* handler: k type=string kind=string quoted=false*/ - - { - - { - if tok != fflib.FFTok_string && tok != fflib.FFTok_null { - return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for string", tok)) - } - } - - if tok == fflib.FFTok_null { - - } else { - - outBuf := fs.Output.Bytes() - - k = string(string(outBuf)) - - } - } - - // Expect ':' after key - tok = fs.Scan() - if tok != fflib.FFTok_colon { - return fs.WrapErr(fmt.Errorf("wanted colon token, but got token: %v", tok)) - } - - tok = fs.Scan() - /* handler: tmpJHeaders type=string kind=string quoted=false*/ - - { - - { - if tok != fflib.FFTok_string && tok != fflib.FFTok_null { - return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for string", tok)) - } - } - - if tok == fflib.FFTok_null { - - } else { - - outBuf := fs.Output.Bytes() - - tmpJHeaders = string(string(outBuf)) - - } - } - - j.Headers[k] = tmpJHeaders - - wantVal = false - } - - } - } - - state = fflib.FFParse_after_value - goto mainparse - -handle_Tag: - - /* handler: j.Tag type=string kind=string quoted=false*/ - - { - - { - if tok != fflib.FFTok_string && tok != fflib.FFTok_null { - return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for string", tok)) - } - } - - if tok == fflib.FFTok_null { - - } else { - - outBuf := fs.Output.Bytes() - - j.Tag = string(string(outBuf)) - - } - } - - state = fflib.FFParse_after_value - goto mainparse - -handle_Body: - - /* handler: j.Body type=string kind=string quoted=false*/ - - { - - { - if tok != fflib.FFTok_string && tok != fflib.FFTok_null { - return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for string", tok)) - } - } - - if tok == fflib.FFTok_null { - - } else { - - outBuf := fs.Output.Bytes() - - j.Body = string(string(outBuf)) - - } - } - - state = fflib.FFParse_after_value - goto mainparse - -wantedvalue: - return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok)) -wrongtokenerror: - return fs.WrapErr(fmt.Errorf("ffjson: wanted token: %v, but got token: %v output=%s", wantedTok, tok, fs.Output.String())) -tokerror: - if fs.BigError != nil { - return fs.WrapErr(fs.BigError) - } - err = fs.Error.ToError() - if err != nil { - return fs.WrapErr(err) - } - panic("ffjson-generated: unreachable, please report bug.") -done: - - return nil -} diff --git a/components/providers/http/decoders/jsonline/data_test.go b/components/providers/http/decoders/jsonline/data_test.go deleted file mode 100644 index f38fc9b34..000000000 --- a/components/providers/http/decoders/jsonline/data_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package jsonline - -import ( - "net/http" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestToRequest(t *testing.T) { - type want struct { - method string - url string - header http.Header - tag string - body []byte - } - var tests = []struct { - name string - json []byte - confHeader http.Header - want want - wantErr bool - }{ - { - name: "GET request", - json: []byte(`{"host": "ya.ru", "method": "GET", "uri": "/00", "tag": "tag", "headers": {"A": "a", "B": "b"}}`), - confHeader: http.Header{"Default": []string{"def"}}, - want: want{"GET", "http://ya.ru/00", http.Header{"Default": []string{"def"}, "A": []string{"a"}, "B": []string{"b"}}, "tag", nil}, - wantErr: false, - }, - { - name: "POST request", - json: []byte(`{"host": "ya.ru", "method": "POST", "uri": "/01?sleep=10", "tag": "tag", "headers": {"A": "a", "B": "b"}, "body": "body"}`), - confHeader: http.Header{"Default": []string{"def"}}, - want: want{"POST", "http://ya.ru/01?sleep=10", http.Header{"Default": []string{"def"}, "A": []string{"a"}, "B": []string{"b"}}, "tag", []byte(`body`)}, - wantErr: false, - }, - { - name: "POST request with json", - json: []byte(`{"host": "ya.ru", "method": "POST", "uri": "/01?sleep=10", "tag": "tag", "headers": {"A": "a", "B": "b"}, "body": "{\"field\":\"value\"}"}`), - confHeader: http.Header{"Default": []string{"def"}}, - want: want{"POST", "http://ya.ru/01?sleep=10", http.Header{"Default": []string{"def"}, "A": []string{"a"}, "B": []string{"b"}}, "tag", []byte(`{"field":"value"}`)}, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert := assert.New(t) - method, url, header, tag, body, err := DecodeAmmo(tt.json, tt.confHeader) - if tt.wantErr { - require.Error(t, err) - return - } - actual := want{method, url, header, tag, body} - assert.NoError(err) - assert.Equal(tt.want, actual) - }) - } -} diff --git a/components/providers/http/decoders/jsonline_test.go b/components/providers/http/decoders/jsonline_test.go index 5d84b4a41..37bc0b14f 100644 --- a/components/providers/http/decoders/jsonline_test.go +++ b/components/providers/http/decoders/jsonline_test.go @@ -1,6 +1,7 @@ package decoders import ( + "bytes" "context" "net/http" "strings" @@ -11,15 +12,96 @@ import ( "github.com/stretchr/testify/require" "github.com/yandex/pandora/components/providers/http/config" "github.com/yandex/pandora/components/providers/http/decoders/ammo" + "github.com/yandex/pandora/lib/pointer" ) -const jsonlineDecoderInput = `{"host": "ya.net", "method": "GET", "uri": "/?sleep=100", "tag": "sleep1", "headers": {"User-agent": "Tank", "Connection": "close"}} +const ( + jsonlineDecoderInput = `{"host": "ya.net", "method": "GET", "uri": "/?sleep=100", "tag": "sleep1", "headers": {"User-agent": "Tank", "Connection": "close"}} {"host": "ya.net", "method": "POST", "uri": "/?sleep=200", "tag": "sleep2", "headers": {"User-agent": "Tank", "Connection": "close"}, "body": "body_data"} {"host": "ya.net", "method": "PUT", "uri": "/", "tag": "sleep3", "headers": {"User-agent": "Tank", "Connection": "close"}, "body": "body_data"} ` + jsonlineDecoderMultiInput = ` + +{ + "host": "ya.net", + "method": "GET", + "uri": "/?sleep=100", + "tag": "sleep1", + "headers": { + "User-agent": "Tank", + "Connection": "close" + } +} +{ + "host": "ya.net", + "method": "POST", + "uri": "/?sleep=200", + "tag": "sleep2", + "headers": { + "User-agent": "Tank", + "Connection": "close" + }, + "body": "body_data" +} + +{ + "host": "ya.net", + "method": "PUT", + "uri": "/", + "tag": "sleep3", + "headers": { + "User-agent": "Tank", + "Connection": "close" + }, + "body": "body_data" +} + +` + + jsonlineDecoderArrayInput = ` + +[ + { + "host": "ya.net", + "method": "GET", + "uri": "/?sleep=100", + "tag": "sleep1", + "headers": { + "User-agent": "Tank", + "Connection": "close" + } + }, + { + "host": "ya.net", + "method": "POST", + "uri": "/?sleep=200", + "tag": "sleep2", + "headers": { + "User-agent": "Tank", + "Connection": "close" + }, + "body": "body_data" + }, + { + "host": "ya.net", + "method": "PUT", + "uri": "/", + "tag": "sleep3", + "headers": { + "User-agent": "Tank", + "Connection": "close" + }, + "body": "body_data" + } +] + + +` +) + func getJsonlineAmmoWants(t *testing.T) []DecodedAmmo { var mustNewAmmo = func(t *testing.T, method string, url string, body []byte, header http.Header, tag string) *ammo.Ammo { a := ammo.Ammo{} @@ -53,32 +135,108 @@ func getJsonlineAmmoWants(t *testing.T) []DecodedAmmo { } func Test_jsonlineDecoder_Scan(t *testing.T) { - decoder := newJsonlineDecoder(strings.NewReader(jsonlineDecoderInput), config.Config{ - Limit: 6, - }, http.Header{"Content-Type": []string{"application/json"}}) + cases := []struct { + name string + input string + wants []DecodedAmmo + }{ + { + name: "default", + input: jsonlineDecoderInput, + wants: getJsonlineAmmoWants(t), + }, + { + name: "multiline json", + input: jsonlineDecoderMultiInput, + wants: getJsonlineAmmoWants(t), + }, + { + name: "array json", + input: jsonlineDecoderArrayInput, + wants: getJsonlineAmmoWants(t), + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + decoder, err := newJsonlineDecoder(strings.NewReader(tt.input), config.Config{ + Limit: 6, + }, http.Header{"Content-Type": []string{"application/json"}}) + require.NoError(t, err) - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() - wants := getJsonlineAmmoWants(t) - for j := 0; j < 2; j++ { - for i, want := range wants { - ammo, err := decoder.Scan(ctx) - assert.NoError(t, err, "iteration %d-%d", j, i) - assert.Equal(t, want, ammo, "iteration %d-%d", j, i) - } + for j := 0; j < 2; j++ { + for i, want := range tt.wants { + ammo, err := decoder.Scan(ctx) + assert.NoError(t, err, "iteration %d-%d", j, i) + assert.Equal(t, want, ammo, "iteration %d-%d", j, i) + } + } + + _, err = decoder.Scan(ctx) + assert.Equal(t, ErrAmmoLimit, err) + assert.Equal(t, uint(len(tt.wants)*2), decoder.ammoNum) + if tt.name == "array json" { + assert.Equal(t, uint(2), decoder.passNum) + } else { + assert.Equal(t, uint(1), decoder.passNum) + } + }) } +} - _, err := decoder.Scan(ctx) - assert.Equal(t, err, ErrAmmoLimit) - assert.Equal(t, decoder.ammoNum, uint(len(wants)*2)) - assert.Equal(t, decoder.passNum, uint(1)) +func Test_jsonlineDecoder_Scan_PassesOnce(t *testing.T) { + cases := []struct { + name string + input string + wants []DecodedAmmo + }{ + { + name: "default", + input: jsonlineDecoderInput, + wants: getJsonlineAmmoWants(t), + }, + { + name: "multiline json", + input: jsonlineDecoderMultiInput, + wants: getJsonlineAmmoWants(t), + }, + { + name: "array json", + input: jsonlineDecoderArrayInput, + wants: getJsonlineAmmoWants(t), + }, + } + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + decoder, err := newJsonlineDecoder(strings.NewReader(tt.input), config.Config{ + Passes: 1, + }, http.Header{"Content-Type": []string{"application/json"}}) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + for i, want := range tt.wants { + ammo, err := decoder.Scan(ctx) + assert.NoError(t, err, "iteration %d", i) + assert.Equal(t, want, ammo, "iteration %d", i) + } + + _, err = decoder.Scan(ctx) + assert.Equal(t, ErrPassLimit, err) + assert.Equal(t, uint(len(tt.wants)), decoder.ammoNum) + assert.Equal(t, uint(1), decoder.passNum) + }) + } } func Test_jsonlineDecoder_LoadAmmo(t *testing.T) { - decoder := newJsonlineDecoder(strings.NewReader(jsonlineDecoderInput), config.Config{ + decoder, err := newJsonlineDecoder(strings.NewReader(jsonlineDecoderInput), config.Config{ Limit: 7, }, http.Header{"Content-Type": []string{"application/json"}}) + require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -92,16 +250,95 @@ func Test_jsonlineDecoder_LoadAmmo(t *testing.T) { assert.Equal(t, decoder.config.Passes, uint(0)) } -func Benchmark_jsonlineDecoder_Scan_line(b *testing.B) { - decoder := newJsonlineDecoder( +func TestIsArray(t *testing.T) { + tests := []struct { + name string + input string + want bool + wantErr *string + }{ + { + name: "empty json array", + input: " [] ", + want: true, + wantErr: nil, + }, + { + name: "non-empty json array", + input: ` [{"name": "Alice"}, {"name": "Bob"}] `, + want: true, + wantErr: nil, + }, + { + name: "empty json object", + input: ` { } `, + want: false, + wantErr: nil, + }, + { + name: "non-empty json object", + input: ` {"name": "Alice", "age": 30} `, + want: false, + wantErr: nil, + }, + { + name: "invalid json", + input: `{"name": "Alice",}`, + want: false, + wantErr: nil, + }, + { + name: "invalid json", + input: ` s{"name": "Alice",}`, + want: false, + wantErr: pointer.ToString("invalid character 's' looking for beginning of value"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := bytes.NewReader([]byte(tt.input)) + result, err := isArray(r) + assert.Equal(t, tt.want, result) + if tt.wantErr == nil { + assert.NoError(t, err) + } else { + assert.Equal(t, *tt.wantErr, err.Error()) + } + }) + } +} + +func Benchmark_jsonlineDecoderScan_line(b *testing.B) { + decoder, err := newJsonlineDecoder( strings.NewReader(jsonlineDecoderInput), config.Config{}, http.Header{"Content-Type": []string{"application/json"}}, ) + require.NoError(b, err) + ctx := context.Background() + b.ResetTimer() + for i := 0; i < b.N; i++ { + a, err := decoder.Scan(ctx) + require.NoError(b, err) + _, err = a.BuildRequest() + require.NoError(b, err) + } +} +func Benchmark_jsonlineDecoderScan_multi(b *testing.B) { + decoder, err := newJsonlineDecoder( + strings.NewReader(jsonlineDecoderMultiInput), config.Config{}, + http.Header{"Content-Type": []string{"application/json"}}, + ) + require.NoError(b, err) + + ctx := context.Background() b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := decoder.Scan(ctx) + a, err := decoder.Scan(ctx) + require.NoError(b, err) + _, err = a.BuildRequest() require.NoError(b, err) } } diff --git a/components/providers/http/provider_test.go b/components/providers/http/provider_test.go index 8f387532c..ba40501d6 100644 --- a/components/providers/http/provider_test.go +++ b/components/providers/http/provider_test.go @@ -5,6 +5,8 @@ import ( "testing" "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/yandex/pandora/components/providers/http/config" "github.com/yandex/pandora/components/providers/http/provider" ) @@ -24,18 +26,12 @@ func TestNewProvider_invalidDecoder(t *testing.T) { func TestNewProvider(t *testing.T) { fs := afero.NewMemMapFs() - t.Run("InvalidDecoder", func(t *testing.T) { - }) - tmpFile, err := fs.Create("ammo") - if err != nil { - t.Fatalf("failed to create temp file: %s", err) - } + require.NoError(t, err) defer os.Remove(tmpFile.Name()) - if _, err := tmpFile.Write([]byte("GET / HTTP/1.1\nHost: example.com\n\n")); err != nil { - t.Fatalf("failed to write data to temp file: %s", err) - } + _, err = tmpFile.Write([]byte(" {}\n\n")) // content is important only for jsonDecoder + require.NoError(t, err) cases := []struct { name string @@ -77,42 +73,27 @@ func TestNewProvider(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - providr, err := NewProvider(fs, tc.conf) + provdr, err := NewProvider(fs, tc.conf) if err != nil { t.Fatalf("failed to create provider: %s", err) } - provider, _ := providr.(*provider.Provider) - + p, ok := provdr.(*provider.Provider) + require.True(t, ok) + require.NotNil(t, p) defer func() { - if err := provider.Close(); err != nil { - t.Fatalf("failed to close provider: %s", err) - } + err := p.Close() + require.NoError(t, err) }() - if provider == nil { - t.Fatal("provider is nil") - } + assert.NotNil(t, p.Sink) + assert.Equal(t, fs, p.FS) + assert.Equal(t, tc.expected, p.Config.Decoder) + assert.Equal(t, tc.filePath, p.Config.File) - if provider.Config.Decoder != tc.expected { - t.Errorf("unexpected decoder type: got %s, want %s", provider.Config.Decoder, tc.expected) - } - if provider.Config.File != tc.filePath { - t.Errorf("unexpected file path: got %s, want %s", provider.Config.File, tc.filePath) - } - - if provider.Decoder == nil && tc.expected != "" { + if p.Decoder == nil && tc.expected != "" { t.Error("decoder is nil") } - - if provider.FS != fs { - t.Errorf("unexpected FS: got %v, want %v", provider.FS, fs) - } - - if provider.Sink == nil { - t.Error("sink is nil") - } }) } - } diff --git a/go.mod b/go.mod index 5550f4fd6..e5740c721 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,6 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.27.10 github.com/pkg/errors v0.9.1 - github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 github.com/spf13/afero v1.9.5 github.com/spf13/viper v1.16.0 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index a76e86f16..ae9241f92 100644 --- a/go.sum +++ b/go.sum @@ -877,8 +877,6 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 h1:xoIK0ctDddBMnc74udxJYBqlo9Ylnsp1waqjLsnef20= -github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=