From 7a499e7c46b27bc49ac679c9b8b48a1cc6e8cd16 Mon Sep 17 00:00:00 2001 From: Gang Chen Date: Fri, 4 Oct 2019 10:07:29 +0200 Subject: [PATCH 1/7] Add support for slice in struct --- cmem/encoder.go | 51 ++++++++++++++++++++++++++++++++++++++++++++++--- h5pt_table.go | 3 +++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/cmem/encoder.go b/cmem/encoder.go index dc7d00a..dd20276 100644 --- a/cmem/encoder.go +++ b/cmem/encoder.go @@ -4,6 +4,11 @@ package cmem +// #include "hdf5.h" +// #include +// #include +import "C" + import ( "encoding/binary" "fmt" @@ -26,8 +31,16 @@ func init() { // subsequently write an in memory object to a PacketTable using Append(). type Encoder struct { // Buf contains the encoded data. - Buf []byte - offset int + Buf []byte + offset int + pointerSlice []unsafe.Pointer +} + +// FreeMemory is to free memory from C +func (enc *Encoder) FreeMemory() { + for i := 0; i < len(enc.pointerSlice); i++ { + C.free(enc.pointerSlice[i]) + } } // Encode encodes the value passed as data to binary form stored in []Buf. This @@ -61,7 +74,39 @@ func (enc *Encoder) Encode(data interface{}) error { rt := rv.Type() switch rt.Kind() { - case reflect.Slice, reflect.Array: + case reflect.Slice: + length := C.size_t(reflect.ValueOf(data).Len()) + msize := C.size_t(rv.Index(0).Type().Size() * uintptr(length)) + + // this is variable length data, it should follow hvl_t format + enc.Encode(length) + + pointer := C.malloc(msize) + enc.pointerSlice = append(enc.pointerSlice, pointer) + + C.memset(pointer, 0, msize) + + var tempBuf []byte + for i := 0; i < rv.Len(); i++ { + var myenc Encoder + if err := myenc.Encode(rv.Index(i).Interface()); err != nil { + return err + } + mypad := myenc.offset - len(myenc.Buf) + if mypad > 0 { + myenc.Buf = append(myenc.Buf, make([]byte, mypad)...) + } + + tempBuf = append(tempBuf, myenc.Buf...) + } + // copy contents from temp buf to C memory + C.memcpy(pointer, unsafe.Pointer(&tempBuf[0]), msize) + + enc.Encode(C.size_t(uintptr(pointer))) + + return nil + + case reflect.Array: for i := 0; i < rv.Len(); i++ { if err := enc.Encode(rv.Index(i).Interface()); err != nil { return err diff --git a/h5pt_table.go b/h5pt_table.go index b94d772..1ef1bf6 100644 --- a/h5pt_table.go +++ b/h5pt_table.go @@ -79,6 +79,8 @@ func (t *Table) Append(args ...interface{}) error { } var enc cmem.Encoder + defer enc.FreeMemory() + for _, arg := range args { if err := enc.Encode(arg); err != nil { return err @@ -167,6 +169,7 @@ func createTable(id C.hid_t, name string, dtype *Datatype, chunkSize, compressio if err := checkID(hid); err != nil { return nil, err } + return newPacketTable(hid), nil } From bb12a97b93ee4501f885b3576c0ad973af82693c Mon Sep 17 00:00:00 2001 From: Gang Chen Date: Fri, 4 Oct 2019 11:02:21 +0200 Subject: [PATCH 2/7] fix unittest for slice update --- cmem/encoder.go | 1 + cmem/encoder_test.go | 83 ++++++++++++++++++++++---------------------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/cmem/encoder.go b/cmem/encoder.go index dd20276..a973a57 100644 --- a/cmem/encoder.go +++ b/cmem/encoder.go @@ -72,6 +72,7 @@ func (enc *Encoder) Encode(data interface{}) error { } rt := rv.Type() + fmt.Println("Kind: ", rt.Kind()) switch rt.Kind() { case reflect.Slice: diff --git a/cmem/encoder_test.go b/cmem/encoder_test.go index 9180ab1..eafba2f 100644 --- a/cmem/encoder_test.go +++ b/cmem/encoder_test.go @@ -6,51 +6,50 @@ package cmem import ( "bytes" - "encoding/binary" - "strconv" "testing" + "encoding/binary" ) func TestEncode(t *testing.T) { - for i, tc := range []struct { - v interface{} - want []byte - }{ - { - v: struct { - V1 uint8 - V2 uint64 - V3 uint8 - V4 uint16 - }{1, 2, 3, 4}, - want: []byte{ - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // |........| - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // |........| - 0x03, 0x00, 0x04, 0x00, /* */ // |....| - }, - }, - { - v: []int32{1, 20, 300, 4000, 50000, 600000, 7000000, 8000000}, - want: []byte{ - 0x01, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, // |........| - 0x2c, 0x01, 0x00, 0x00, 0xa0, 0x0f, 0x00, 0x00, // |,.......| - 0x50, 0xc3, 0x00, 0x00, 0xc0, 0x27, 0x09, 0x00, // |P....'..| - 0xc0, 0xcf, 0x6a, 0x00, 0x00, 0x12, 0x7a, 0x00, // |..j...z.| - }, - }, - } { - t.Run(strconv.Itoa(i), func(t *testing.T) { - oldEndian := nativeEndian - nativeEndian = binary.LittleEndian - defer func() { nativeEndian = oldEndian }() - var enc Encoder - err := enc.Encode(tc.v) - if err != nil { - t.Fatalf("could not encode: %v", err) - } - if !bytes.Equal(enc.Buf, tc.want) { - t.Fatalf("encoding error:\ngot = %v\nwant= %v", enc.Buf, tc.want) - } - }) + v := struct { + V1 uint8 + V2 uint64 + V3 uint8 + V4 uint16 + }{1, 2, 3, 4} + want := []byte{ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // |........| + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // |........| + 0x03, 0x00, 0x04, 0x00, /* */ // |....| + } + var enc Encoder + err := enc.Encode(v) + if err != nil { + t.Fatalf("could not encode: %v", err) } + if !bytes.Equal(enc.Buf, want) { + t.Fatalf("encoding error:\ngot = %v\nwant= %v", enc.Buf, want) + } +} + +func TestEncodeSlice(t *testing.T) { + v := []int32{1, 20, 300, 4000, 50000, 600000, 7000000, 8000000} + want := []byte{ + 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // |........| + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // |,.......| + } + var enc Encoder + err := enc.Encode(v) + if err != nil { + t.Fatalf("could not encode: %v", err) + } + pointer := enc.pointerSlice[0] + bs := make([]byte, 8) + binary.LittleEndian.PutUint64(bs, uint64(uintptr(pointer))) + copy(want[8:16], bs[:]) + + if !bytes.Equal(enc.Buf, want) { + t.Fatalf("encoding error:\ngot = %v\nwant= %v", enc.Buf, want) + } + } From 704fe2c5e942881e5be515229d43a566939c60fb Mon Sep 17 00:00:00 2001 From: Gang Chen Date: Fri, 4 Oct 2019 11:04:35 +0200 Subject: [PATCH 3/7] remove unwanted println --- cmem/encoder.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmem/encoder.go b/cmem/encoder.go index a973a57..dd20276 100644 --- a/cmem/encoder.go +++ b/cmem/encoder.go @@ -72,7 +72,6 @@ func (enc *Encoder) Encode(data interface{}) error { } rt := rv.Type() - fmt.Println("Kind: ", rt.Kind()) switch rt.Kind() { case reflect.Slice: From aa5b705c61a90938d03f5eaa535ceb07534c79b4 Mon Sep 17 00:00:00 2001 From: Gang Chen Date: Mon, 7 Oct 2019 08:21:25 +0200 Subject: [PATCH 4/7] remove useless lines --- cmem/encoder.go | 1 - h5pt_table.go | 1 - 2 files changed, 2 deletions(-) diff --git a/cmem/encoder.go b/cmem/encoder.go index dd20276..0c93bed 100644 --- a/cmem/encoder.go +++ b/cmem/encoder.go @@ -4,7 +4,6 @@ package cmem -// #include "hdf5.h" // #include // #include import "C" diff --git a/h5pt_table.go b/h5pt_table.go index 1ef1bf6..287d02f 100644 --- a/h5pt_table.go +++ b/h5pt_table.go @@ -169,7 +169,6 @@ func createTable(id C.hid_t, name string, dtype *Datatype, chunkSize, compressio if err := checkID(hid); err != nil { return nil, err } - return newPacketTable(hid), nil } From 3894b348439619013c82db989887f67e10f2167d Mon Sep 17 00:00:00 2001 From: Gang Chen Date: Mon, 7 Oct 2019 08:28:12 +0200 Subject: [PATCH 5/7] solve ci issues --- cmem/encoder.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmem/encoder.go b/cmem/encoder.go index 0c93bed..e0e3e12 100644 --- a/cmem/encoder.go +++ b/cmem/encoder.go @@ -78,7 +78,9 @@ func (enc *Encoder) Encode(data interface{}) error { msize := C.size_t(rv.Index(0).Type().Size() * uintptr(length)) // this is variable length data, it should follow hvl_t format - enc.Encode(length) + if err := enc.Encode(length); err != nil { + return err + } pointer := C.malloc(msize) enc.pointerSlice = append(enc.pointerSlice, pointer) @@ -101,7 +103,9 @@ func (enc *Encoder) Encode(data interface{}) error { // copy contents from temp buf to C memory C.memcpy(pointer, unsafe.Pointer(&tempBuf[0]), msize) - enc.Encode(C.size_t(uintptr(pointer))) + if err = enc.Encode(C.size_t(uintptr(pointer))); err != nil { + return err + } return nil From 25ef62c0748a9b7fe5c97690a8927871bb49fba8 Mon Sep 17 00:00:00 2001 From: Gang Chen Date: Mon, 7 Oct 2019 08:31:15 +0200 Subject: [PATCH 6/7] solve ci issue --- cmem/encoder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmem/encoder.go b/cmem/encoder.go index e0e3e12..f2e4445 100644 --- a/cmem/encoder.go +++ b/cmem/encoder.go @@ -103,7 +103,7 @@ func (enc *Encoder) Encode(data interface{}) error { // copy contents from temp buf to C memory C.memcpy(pointer, unsafe.Pointer(&tempBuf[0]), msize) - if err = enc.Encode(C.size_t(uintptr(pointer))); err != nil { + if err := enc.Encode(C.size_t(uintptr(pointer))); err != nil { return err } From 33bf8e410dccc1088588602cc0419cfe5c9fb774 Mon Sep 17 00:00:00 2001 From: Gang Chen Date: Mon, 7 Oct 2019 14:57:06 +0200 Subject: [PATCH 7/7] Take tempBuf length as malloc size --- cmem/encoder.go | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/cmem/encoder.go b/cmem/encoder.go index f2e4445..b4fef1c 100644 --- a/cmem/encoder.go +++ b/cmem/encoder.go @@ -74,20 +74,8 @@ func (enc *Encoder) Encode(data interface{}) error { switch rt.Kind() { case reflect.Slice: - length := C.size_t(reflect.ValueOf(data).Len()) - msize := C.size_t(rv.Index(0).Type().Size() * uintptr(length)) - - // this is variable length data, it should follow hvl_t format - if err := enc.Encode(length); err != nil { - return err - } - - pointer := C.malloc(msize) - enc.pointerSlice = append(enc.pointerSlice, pointer) - - C.memset(pointer, 0, msize) - var tempBuf []byte + for i := 0; i < rv.Len(); i++ { var myenc Encoder if err := myenc.Encode(rv.Index(i).Interface()); err != nil { @@ -97,13 +85,20 @@ func (enc *Encoder) Encode(data interface{}) error { if mypad > 0 { myenc.Buf = append(myenc.Buf, make([]byte, mypad)...) } - tempBuf = append(tempBuf, myenc.Buf...) } - // copy contents from temp buf to C memory - C.memcpy(pointer, unsafe.Pointer(&tempBuf[0]), msize) - if err := enc.Encode(C.size_t(uintptr(pointer))); err != nil { + arraySize := C.size_t(len(tempBuf)) + vlArray := C.malloc(arraySize) + C.bzero(vlArray, arraySize) + C.memcpy(vlArray, unsafe.Pointer(&tempBuf[0]), C.size_t(arraySize)) + enc.pointerSlice = append(enc.pointerSlice, vlArray) + + // this is variable length data, it should follow hvl_t format + if err := enc.Encode(C.size_t(rv.Len())); err != nil { + return err + } + if err := enc.Encode(C.size_t(uintptr(vlArray))); err != nil { return err }