From 996499174c1230510df687ac7fb09fe78bc9f97f Mon Sep 17 00:00:00 2001 From: Andrew Brampton Date: Sun, 11 Apr 2021 14:54:12 -0700 Subject: [PATCH 1/6] Added a new codegen option to throw a error on a unknown field. --- errors.go | 18 ++++++++++++++++++ gojay/codegen/options.go | 27 +++++++++++++++------------ gojay/codegen/struct.go | 2 ++ gojay/codegen/template.go | 6 +++++- gojay/gojay.go | 4 +++- 5 files changed, 43 insertions(+), 14 deletions(-) diff --git a/errors.go b/errors.go index 0fd52e6..e9eaddb 100644 --- a/errors.go +++ b/errors.go @@ -86,3 +86,21 @@ func (err InvalidUsagePooledEncoderError) Error() string { // ErrUnmarshalPtrExpected is the error returned when unmarshal expects a pointer value, // When using `dec.ObjectNull` or `dec.ArrayNull` for example. var ErrUnmarshalPtrExpected = errors.New("Cannot unmarshal to given value, a pointer is expected") + +const unknownFieldErrorMsg = "Encountered unknown field %q while unmarshal JSON to type '%T'" + +// UnknownFieldError occurs when Decoding a object with a unknown field. +type UnknownFieldError string + +func (err UnknownFieldError) Error() string { + return string(err) +} +func (dec *Decoder) makeUnknownFieldErr(v interface{}, key string) error { + return UnknownFieldError( + fmt.Sprintf( + unknownFieldErrorMsg, + key, + v, + ), + ) +} diff --git a/gojay/codegen/options.go b/gojay/codegen/options.go index 5d400cd..c5d8b5d 100644 --- a/gojay/codegen/options.go +++ b/gojay/codegen/options.go @@ -10,12 +10,13 @@ import ( ) type Options struct { - Source string - Dest string - Types []string - PoolObjects bool - TagName string - Pkg string + Source string + Dest string + Types []string + PoolObjects bool + TagName string + Pkg string + ErrOnUnknown bool } func (o *Options) Validate() error { @@ -29,12 +30,13 @@ func (o *Options) Validate() error { } const ( - optionKeySource = "s" - optionKeyDest = "o" - optionKeyTypes = "t" - optionKeyTagName = "a" - optionKeyPoolObjects = "p" - optionKeyPkg = "pkg" + optionKeySource = "s" + optionKeyDest = "o" + optionKeyTypes = "t" + optionKeyTagName = "a" + optionKeyPoolObjects = "p" + optionKeyPkg = "pkg" + optionKeyErrOnUnknown = "e" ) //NewOptionsWithFlagSet creates a new options for the supplide flagset @@ -48,6 +50,7 @@ func NewOptionsWithFlagSet(set *flag.FlagSet) *Options { result.TagName = set.Lookup(optionKeyTagName).Value.String() result.Types = strings.Split(set.Lookup(optionKeyTypes).Value.String(), ",") result.Pkg = set.Lookup(optionKeyPkg).Value.String() + result.ErrOnUnknown = toolbox.AsBoolean(set.Lookup(optionKeyErrOnUnknown).Value.String()) if result.Source == "" { result.Source = url.NewResource(".").ParsedURL.Path } diff --git a/gojay/codegen/struct.go b/gojay/codegen/struct.go index 57512a7..bf44943 100644 --- a/gojay/codegen/struct.go +++ b/gojay/codegen/struct.go @@ -46,6 +46,7 @@ func (s *Struct) generateEncoding(structInfo *toolbox.TypeInfo) (string, error) DecodingCases string Reset string FieldCount int + ErrOnUnknown bool }{ Receiver: s.Alias + " *" + s.Name, DecodingCases: strings.Join(decodingCases, "\n"), @@ -54,6 +55,7 @@ func (s *Struct) generateEncoding(structInfo *toolbox.TypeInfo) (string, error) InitEmbedded: initEmbedded, Reset: resetCode, Alias: s.Alias, + ErrOnUnknown: s.options.ErrOnUnknown, } return expandBlockTemplate(encodingStructType, data) } diff --git a/gojay/codegen/template.go b/gojay/codegen/template.go index 6b1ddb6..fa93257 100644 --- a/gojay/codegen/template.go +++ b/gojay/codegen/template.go @@ -174,7 +174,11 @@ func ({{.Receiver}}) IsNil() bool { func ({{.Receiver}}) UnmarshalJSONObject(dec *gojay.Decoder, k string) error { {{.InitEmbedded}} switch k { -{{.DecodingCases}} +{{.DecodingCases}} +{{if .ErrOnUnknown}} + default: + return dec.makeUnknownFieldErr({{.Alias}}, k) +{{end}} } return nil } diff --git a/gojay/gojay.go b/gojay/gojay.go index e9358cb..0dcf78c 100644 --- a/gojay/gojay.go +++ b/gojay/gojay.go @@ -2,8 +2,9 @@ package main import ( "flag" - "github.com/francoispqt/gojay/gojay/codegen" "log" + + "github.com/francoispqt/gojay/gojay/codegen" ) var pkg = flag.String("pkg", "", "the package name of the generated file") @@ -12,6 +13,7 @@ var src = flag.String("s", "", "source dir or file (absolute or relative path)") var types = flag.String("t", "", "types to generate") var annotation = flag.String("a", "json", "annotation tag (default json)") var poolObjects = flag.String("p", "", "generate code to reuse objects using sync.Pool") +var errOnUnknown = flag.String("e", "", "generate code to error on unknown fields") func main() { flag.Parse() From 387bc05695cf5792b56df7fd458433d3a526dd25 Mon Sep 17 00:00:00 2001 From: Andrew Brampton Date: Sun, 11 Apr 2021 15:02:01 -0700 Subject: [PATCH 2/6] Add documentation for new flag. --- gojay/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gojay/README.md b/gojay/README.md index fb0f7f0..e22d466 100644 --- a/gojay/README.md +++ b/gojay/README.md @@ -23,7 +23,11 @@ If you just want to the output to stdout, omit the -o flag. - t Types to generate with all its dependencies (comma separated) - a Annotation tag used to read metadata (default: json) - o Output file (relative or absolute path) + +Flags that alter the generated code: + - p Pool to reuse object (using sync.Pool) +- e Return a error on unknown fields (default: false) Examples: From e8afb975f912a4095f92713314e274a6080db842 Mon Sep 17 00:00:00 2001 From: Andrew Brampton Date: Sun, 11 Apr 2021 15:02:15 -0700 Subject: [PATCH 3/6] Fixed a couple of minor typos. --- gojay/codegen/generator_test.go | 7 ++++--- gojay/codegen/template.go | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/gojay/codegen/generator_test.go b/gojay/codegen/generator_test.go index 44ecfde..bea81c8 100644 --- a/gojay/codegen/generator_test.go +++ b/gojay/codegen/generator_test.go @@ -1,11 +1,12 @@ package codegen import ( - "github.com/stretchr/testify/assert" - "github.com/viant/toolbox" "log" "path" "testing" + + "github.com/stretchr/testify/assert" + "github.com/viant/toolbox" ) func TestGenerator_Generate(t *testing.T) { @@ -45,7 +46,7 @@ func TestGenerator_Generate(t *testing.T) { }, }, { - description: "struct with json annotation and time/foarmat|layouat generation", + description: "struct with json annotation and time/format|layout generation", options: &Options{ Source: path.Join(parent, "annotated_struct"), Types: []string{"Message"}, diff --git a/gojay/codegen/template.go b/gojay/codegen/template.go index fa93257..7bf7546 100644 --- a/gojay/codegen/template.go +++ b/gojay/codegen/template.go @@ -309,7 +309,7 @@ func expandTemplate(namespace string, dictionary map[int]string, key int, data i } temlate, err := template.New(id).Parse(textTemplate) if err != nil { - return "", fmt.Errorf("fiailed to parse template %v %v, due to %v", namespace, key, err) + return "", fmt.Errorf("failed to parse template %v %v, due to %v", namespace, key, err) } writer := new(bytes.Buffer) err = temlate.Execute(writer, data) From 66e78dc38b6ab3645675a2f1a358ae8972304c93 Mon Sep 17 00:00:00 2001 From: Andrew Brampton Date: Sun, 11 Apr 2021 15:47:11 -0700 Subject: [PATCH 4/6] Changed the unknown field error to be a public method (so we can actually use it in generated code). --- errors.go | 2 +- gojay/codegen/template.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/errors.go b/errors.go index e9eaddb..fdbba08 100644 --- a/errors.go +++ b/errors.go @@ -95,7 +95,7 @@ type UnknownFieldError string func (err UnknownFieldError) Error() string { return string(err) } -func (dec *Decoder) makeUnknownFieldErr(v interface{}, key string) error { +func MakeUnknownFieldErr(v interface{}, key string) error { return UnknownFieldError( fmt.Sprintf( unknownFieldErrorMsg, diff --git a/gojay/codegen/template.go b/gojay/codegen/template.go index 7bf7546..7c4b019 100644 --- a/gojay/codegen/template.go +++ b/gojay/codegen/template.go @@ -177,7 +177,7 @@ func ({{.Receiver}}) UnmarshalJSONObject(dec *gojay.Decoder, k string) error { {{.DecodingCases}} {{if .ErrOnUnknown}} default: - return dec.makeUnknownFieldErr({{.Alias}}, k) + return gojay.MakeUnknownFieldErr({{.Alias}}, k) {{end}} } return nil From 0dfd20f25d82cbdbab9ba54e5fb2632dce9717df Mon Sep 17 00:00:00 2001 From: Andrew Brampton Date: Sun, 11 Apr 2021 15:47:51 -0700 Subject: [PATCH 5/6] Add unknown field tests. --- gojay/codegen/generator_test.go | 9 +++++ gojay/codegen/test/unknown_struct/encoding.go | 40 +++++++++++++++++++ .../test/unknown_struct/encoding_test.go | 27 +++++++++++++ gojay/codegen/test/unknown_struct/message.go | 6 +++ 4 files changed, 82 insertions(+) create mode 100644 gojay/codegen/test/unknown_struct/encoding.go create mode 100644 gojay/codegen/test/unknown_struct/encoding_test.go create mode 100644 gojay/codegen/test/unknown_struct/message.go diff --git a/gojay/codegen/generator_test.go b/gojay/codegen/generator_test.go index bea81c8..9b030a5 100644 --- a/gojay/codegen/generator_test.go +++ b/gojay/codegen/generator_test.go @@ -55,6 +55,15 @@ func TestGenerator_Generate(t *testing.T) { TagName: "json", }, }, + { + description: "basic struct code generation with unknown field errors", + options: &Options{ + Source: path.Join(parent, "unknown_struct"), + Types: []string{"Message"}, + Dest: path.Join(parent, "unknown_struct", "encoding.go"), + ErrOnUnknown: true, + }, + }, } for _, useCase := range useCases { diff --git a/gojay/codegen/test/unknown_struct/encoding.go b/gojay/codegen/test/unknown_struct/encoding.go new file mode 100644 index 0000000..5414f49 --- /dev/null +++ b/gojay/codegen/test/unknown_struct/encoding.go @@ -0,0 +1,40 @@ +// Code generated by Gojay. DO NOT EDIT. + +package unknown_struct + +import ( + "github.com/francoispqt/gojay" +) + +// MarshalJSONObject implements MarshalerJSONObject +func (m *Message) MarshalJSONObject(enc *gojay.Encoder) { + enc.IntKey("Id", m.Id) + enc.StringKey("Name", m.Name) +} + +// IsNil checks if instance is nil +func (m *Message) IsNil() bool { + return m == nil +} + +// UnmarshalJSONObject implements gojay's UnmarshalerJSONObject +func (m *Message) UnmarshalJSONObject(dec *gojay.Decoder, k string) error { + + switch k { + case "Id": + return dec.Int(&m.Id) + + case "Name": + return dec.String(&m.Name) + + default: + return gojay.MakeUnknownFieldErr(m, k) + + } + return nil +} + +// NKeys returns the number of keys to unmarshal +func (m *Message) NKeys() int { + return 0 // Unmarshal all keys so we can catch trailing unknown keys. +} diff --git a/gojay/codegen/test/unknown_struct/encoding_test.go b/gojay/codegen/test/unknown_struct/encoding_test.go new file mode 100644 index 0000000..e0b3779 --- /dev/null +++ b/gojay/codegen/test/unknown_struct/encoding_test.go @@ -0,0 +1,27 @@ +package unknown_struct + +import ( + "testing" + + "github.com/francoispqt/gojay" + "github.com/stretchr/testify/require" +) + +var msg = &Message{ + Id: 1022, + Name: "name acc", +} + +var jsonData = `{ + "Id": 1022, + "Name": "name acc", + "Price": 13.3 +}` + +func TestMessage_Unmarshal(t *testing.T) { + var err error + var data = []byte(jsonData) + message := &Message{} + err = gojay.UnmarshalJSONObject(data, message) + require.EqualError(t, err, gojay.MakeUnknownFieldErr(message, "Price").Error()) +} diff --git a/gojay/codegen/test/unknown_struct/message.go b/gojay/codegen/test/unknown_struct/message.go new file mode 100644 index 0000000..cc8a70d --- /dev/null +++ b/gojay/codegen/test/unknown_struct/message.go @@ -0,0 +1,6 @@ +package unknown_struct + +type Message struct { + Id int + Name string +} From 0b6bcab04741618853acc1710b6708aca374ecd9 Mon Sep 17 00:00:00 2001 From: Andrew Brampton Date: Sun, 11 Apr 2021 15:48:05 -0700 Subject: [PATCH 6/6] Change the NKeys behaviour when checking for unknown fields. Specifically always check for all keys, instead of the first N keys. --- gojay/codegen/template.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/gojay/codegen/template.go b/gojay/codegen/template.go index 7c4b019..7d32cd2 100644 --- a/gojay/codegen/template.go +++ b/gojay/codegen/template.go @@ -184,7 +184,13 @@ func ({{.Receiver}}) UnmarshalJSONObject(dec *gojay.Decoder, k string) error { } // NKeys returns the number of keys to unmarshal -func ({{.Receiver}}) NKeys() int { return {{.FieldCount}} } +func ({{.Receiver}}) NKeys() int { +{{- if .ErrOnUnknown}} + return 0 // Unmarshal all keys so we can catch trailing unknown keys. +{{else}} + return {{.FieldCount}} +{{end -}} +} {{.Reset}}