From f4628b433e1dc53d56f7f71f37ba835d717b0769 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 22 Aug 2023 17:59:42 -0400 Subject: [PATCH 01/11] prepare a contiguous memory block for messages, without additional copying --- crypto/batchverifier.go | 25 +++++++++++++++++-------- crypto/onetimesig.go | 12 +++++++++++- crypto/util.go | 14 ++++++++++++++ 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index 9c14771bac..caf4a73b89 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -123,11 +123,16 @@ func (b *BatchVerifier) VerifyWithFeedback() (failed []bool, err error) { if b.GetNumberOfEnqueuedSignatures() == 0 { return nil, nil } - var messages = make([][]byte, b.GetNumberOfEnqueuedSignatures()) + msgLengths := make([]int, 0, len(b.messages)) + var messages = make([]byte, 0, b.GetNumberOfEnqueuedSignatures()*64) + + lenWas := 0 for i := range b.messages { - messages[i] = HashRep(b.messages[i]) + messages = HashRepToBuff(b.messages[i], messages) + msgLengths = append(msgLengths, len(messages)-lenWas) + lenWas = len(messages) } - allValid, failed := batchVerificationImpl(messages, b.publicKeys, b.signatures) + allValid, failed := batchVerificationImpl(messages, msgLengths, b.publicKeys, b.signatures) if allValid { return failed, nil } @@ -137,9 +142,9 @@ func (b *BatchVerifier) VerifyWithFeedback() (failed []bool, err error) { // batchVerificationImpl invokes the ed25519 batch verification algorithm. // it returns true if all the signatures were authentically signed by the owners // otherwise, returns false, and sets the indexes of the failed sigs in failed -func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, signatures []Signature) (allSigsValid bool, failed []bool) { +func batchVerificationImpl(messages []byte, msgLengths []int, publicKeys []SignatureVerifier, signatures []Signature) (allSigsValid bool, failed []bool) { - numberOfSignatures := len(messages) + numberOfSignatures := len(msgLengths) messagesAllocation := C.malloc(C.size_t(C.sizeofPtr * numberOfSignatures)) messagesLenAllocation := C.malloc(C.size_t(C.sizeofULongLong * numberOfSignatures)) @@ -156,10 +161,14 @@ func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, si C.free(valid) }() + // Pin messages, publicKeys and signatures here. remove KeepAlive calls below + // load all the data pointers into the array pointers. + mPos := 0 for i := 0; i < numberOfSignatures; i++ { - *(*uintptr)(unsafe.Pointer(uintptr(messagesAllocation) + uintptr(i*C.sizeofPtr))) = uintptr(unsafe.Pointer(&messages[i][0])) - *(*C.ulonglong)(unsafe.Pointer(uintptr(messagesLenAllocation) + uintptr(i*C.sizeofULongLong))) = C.ulonglong(len(messages[i])) + *(*uintptr)(unsafe.Pointer(uintptr(messagesAllocation) + uintptr(i*C.sizeofPtr))) = uintptr(unsafe.Pointer(&messages[mPos])) + mPos = mPos + msgLengths[i] + *(*C.ulonglong)(unsafe.Pointer(uintptr(messagesLenAllocation) + uintptr(i*C.sizeofULongLong))) = C.ulonglong(msgLengths[i]) *(*uintptr)(unsafe.Pointer(uintptr(publicKeysAllocation) + uintptr(i*C.sizeofPtr))) = uintptr(unsafe.Pointer(&publicKeys[i][0])) *(*uintptr)(unsafe.Pointer(uintptr(signaturesAllocation) + uintptr(i*C.sizeofPtr))) = uintptr(unsafe.Pointer(&signatures[i][0])) } @@ -170,7 +179,7 @@ func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, si (*C.ulonglong)(unsafe.Pointer(messagesLenAllocation)), (**C.uchar)(unsafe.Pointer(publicKeysAllocation)), (**C.uchar)(unsafe.Pointer(signaturesAllocation)), - C.size_t(len(messages)), + C.size_t(numberOfSignatures), (*C.int)(unsafe.Pointer(valid))) runtime.KeepAlive(messages) diff --git a/crypto/onetimesig.go b/crypto/onetimesig.go index 344fd33f77..c4f9fef5cd 100644 --- a/crypto/onetimesig.go +++ b/crypto/onetimesig.go @@ -319,8 +319,18 @@ func (v OneTimeSignatureVerifier) Verify(id OneTimeSignatureIdentifier, message Batch: id.Batch, } + msgLengths := make([]int, 0, 3) + estimatedSize := 256 + messageBuffer := make([]byte, 0, estimatedSize) + messageBuffer = HashRepToBuff(batchID, messageBuffer) + msgLengths = append(msgLengths, len(messageBuffer)) + messageBuffer = HashRepToBuff(offsetID, messageBuffer) + msgLengths = append(msgLengths, len(messageBuffer)-msgLengths[0]) + messageBuffer = HashRepToBuff(message, messageBuffer) + msgLengths = append(msgLengths, len(messageBuffer)-msgLengths[1]-msgLengths[0]) allValid, _ := batchVerificationImpl( - [][]byte{HashRep(batchID), HashRep(offsetID), HashRep(message)}, + messageBuffer, + msgLengths, []PublicKey{PublicKey(v), PublicKey(batchID.SubKeyPK), PublicKey(offsetID.SubKeyPK)}, []Signature{Signature(sig.PK2Sig), Signature(sig.PK1Sig), Signature(sig.Sig)}, ) diff --git a/crypto/util.go b/crypto/util.go index 60bb12aef0..19b28c99ea 100644 --- a/crypto/util.go +++ b/crypto/util.go @@ -40,6 +40,20 @@ func HashRep(h Hashable) []byte { return append([]byte(hashid), data...) } +// HashRepToBuff appends the correct hashid before the message to be hashed into the provided buffer +func HashRepToBuff(h Hashable, buffer []byte) []byte { + hashid, data := h.ToBeHashed() + neededCapacity := len(buffer) + len((string(hashid))) + len(data) + if cap(buffer) < neededCapacity { + newBuffer := make([]byte, 0, neededCapacity) + newBuffer = append(newBuffer, buffer...) + buffer = newBuffer + } + buffer = append(buffer, hashid...) + buffer = append(buffer, data...) + return buffer +} + // DigestSize is the number of bytes in the preferred hash Digest used here. const DigestSize = sha512.Size256 From 7ab3c2792bcbbc0f156e1937abc4bfd451dc02a5 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 25 Aug 2023 01:25:26 -0400 Subject: [PATCH 02/11] use wrapper C function without using byte arrays for signatures --- crypto/batchverifier.go | 57 ++++++++++++----------------------------- 1 file changed, 17 insertions(+), 40 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index caf4a73b89..149e992008 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -30,15 +30,19 @@ package crypto // #cgo windows,amd64 CFLAGS: -I${SRCDIR}/libs/windows/amd64/include // #cgo windows,amd64 LDFLAGS: ${SRCDIR}/libs/windows/amd64/lib/libsodium.a // #include -// #include "sodium.h" // enum { // sizeofPtr = sizeof(void*), // sizeofULongLong = sizeof(unsigned long long), // }; +// int ed25519_batch_wrapper(const unsigned char *messages1D, +// const unsigned long long *mlen, +// const unsigned char *publicKeys1D, +// const unsigned char *signatures1D, +// size_t num, +// int *valid_p); import "C" import ( "errors" - "runtime" "unsafe" ) @@ -145,51 +149,24 @@ func (b *BatchVerifier) VerifyWithFeedback() (failed []bool, err error) { func batchVerificationImpl(messages []byte, msgLengths []int, publicKeys []SignatureVerifier, signatures []Signature) (allSigsValid bool, failed []bool) { numberOfSignatures := len(msgLengths) - - messagesAllocation := C.malloc(C.size_t(C.sizeofPtr * numberOfSignatures)) - messagesLenAllocation := C.malloc(C.size_t(C.sizeofULongLong * numberOfSignatures)) - publicKeysAllocation := C.malloc(C.size_t(C.sizeofPtr * numberOfSignatures)) - signaturesAllocation := C.malloc(C.size_t(C.sizeofPtr * numberOfSignatures)) - valid := C.malloc(C.size_t(C.sizeof_int * numberOfSignatures)) - - defer func() { - // release staging memory - C.free(messagesAllocation) - C.free(messagesLenAllocation) - C.free(publicKeysAllocation) - C.free(signaturesAllocation) - C.free(valid) - }() - - // Pin messages, publicKeys and signatures here. remove KeepAlive calls below - - // load all the data pointers into the array pointers. - mPos := 0 - for i := 0; i < numberOfSignatures; i++ { - *(*uintptr)(unsafe.Pointer(uintptr(messagesAllocation) + uintptr(i*C.sizeofPtr))) = uintptr(unsafe.Pointer(&messages[mPos])) - mPos = mPos + msgLengths[i] - *(*C.ulonglong)(unsafe.Pointer(uintptr(messagesLenAllocation) + uintptr(i*C.sizeofULongLong))) = C.ulonglong(msgLengths[i]) - *(*uintptr)(unsafe.Pointer(uintptr(publicKeysAllocation) + uintptr(i*C.sizeofPtr))) = uintptr(unsafe.Pointer(&publicKeys[i][0])) - *(*uintptr)(unsafe.Pointer(uintptr(signaturesAllocation) + uintptr(i*C.sizeofPtr))) = uintptr(unsafe.Pointer(&signatures[i][0])) + messageLens := make([]C.ulonglong, len(msgLengths)) + for i, l := range msgLengths { + messageLens[i] = C.ulonglong(l) } + valid := make([]C.int, numberOfSignatures) // call the batch verifier - allValid := C.crypto_sign_ed25519_open_batch( - (**C.uchar)(unsafe.Pointer(messagesAllocation)), - (*C.ulonglong)(unsafe.Pointer(messagesLenAllocation)), - (**C.uchar)(unsafe.Pointer(publicKeysAllocation)), - (**C.uchar)(unsafe.Pointer(signaturesAllocation)), + allValid := C.ed25519_batch_wrapper( + (*C.uchar)(&messages[0]), + (*C.ulonglong)(&messageLens[0]), + (*C.uchar)(&(publicKeys[0][0])), + (*C.uchar)(&(signatures[0][0])), C.size_t(numberOfSignatures), - (*C.int)(unsafe.Pointer(valid))) - - runtime.KeepAlive(messages) - runtime.KeepAlive(publicKeys) - runtime.KeepAlive(signatures) + (*C.int)(&valid[0])) failed = make([]bool, numberOfSignatures) for i := 0; i < numberOfSignatures; i++ { - cint := *(*C.int)(unsafe.Pointer(uintptr(valid) + uintptr(i*C.sizeof_int))) - failed[i] = (cint == 0) + failed[i] = (valid[i] == 0) } return allValid == 0, failed } From a4a5d31196f5a5639d9c8e1660da3c4b472d73e3 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 25 Aug 2023 01:33:57 -0400 Subject: [PATCH 03/11] add C wrapper from the POC --- crypto/batchverifier.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 crypto/batchverifier.c diff --git a/crypto/batchverifier.c b/crypto/batchverifier.c new file mode 100644 index 0000000000..8fd233ab8f --- /dev/null +++ b/crypto/batchverifier.c @@ -0,0 +1,28 @@ +#include "sodium.h" +int ed25519_batch_wrapper(const unsigned char *messages1D, + const unsigned long long *mlen, + const unsigned char *publicKeys1D, + const unsigned char *signatures1D, + size_t num, + int *valid) { + int ret; + const unsigned char **messages2D, **publicKeys2D, **signatures2D; + messages2D = malloc(num * sizeof(unsigned char *)); + publicKeys2D = malloc(num * sizeof(unsigned char *)); + signatures2D = malloc(num * sizeof(unsigned char *)); + + // fill 2-D arrays for messages, pks, sigs from provided 1-D arrays + unsigned long long mpos = 0; + for (size_t i = 0; i < num; i++) { + messages2D[i] = &messages1D[mpos]; + mpos += mlen[i]; + publicKeys2D[i] = &publicKeys1D[i*crypto_sign_ed25519_PUBLICKEYBYTES]; + signatures2D[i] = &signatures1D[i*crypto_sign_ed25519_BYTES]; + } + ret = crypto_sign_ed25519_open_batch(messages2D, mlen, publicKeys2D, signatures2D, num, valid); + + free(messages2D); + free(publicKeys2D); + free(signatures2D); + return ret; +} From dc241bba13106cfe27e4820ea69509c2fce4c401 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Fri, 25 Aug 2023 14:58:11 -0400 Subject: [PATCH 04/11] keep the keepAlive until it is replaced by Pin --- crypto/batchverifier.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index 149e992008..eb0ffce20c 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -43,6 +43,7 @@ package crypto import "C" import ( "errors" + "runtime" "unsafe" ) @@ -164,6 +165,12 @@ func batchVerificationImpl(messages []byte, msgLengths []int, publicKeys []Signa C.size_t(numberOfSignatures), (*C.int)(&valid[0])) + // These calls will be replaced with Pin in the next Go version upgrade + runtime.KeepAlive(messages) + runtime.KeepAlive(messageLens) + runtime.KeepAlive(publicKeys) + runtime.KeepAlive(signatures) + failed = make([]bool, numberOfSignatures) for i := 0; i < numberOfSignatures; i++ { failed[i] = (valid[i] == 0) From b40b126dfe2e3277d3587b92c9b6af335cfb490e Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 28 Aug 2023 14:09:10 -0400 Subject: [PATCH 05/11] add a test and simplify HashRepToBuff --- crypto/util.go | 6 ------ crypto/util_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/crypto/util.go b/crypto/util.go index 19b28c99ea..90bd95d7e9 100644 --- a/crypto/util.go +++ b/crypto/util.go @@ -43,12 +43,6 @@ func HashRep(h Hashable) []byte { // HashRepToBuff appends the correct hashid before the message to be hashed into the provided buffer func HashRepToBuff(h Hashable, buffer []byte) []byte { hashid, data := h.ToBeHashed() - neededCapacity := len(buffer) + len((string(hashid))) + len(data) - if cap(buffer) < neededCapacity { - newBuffer := make([]byte, 0, neededCapacity) - newBuffer = append(newBuffer, buffer...) - buffer = newBuffer - } buffer = append(buffer, hashid...) buffer = append(buffer, data...) return buffer diff --git a/crypto/util_test.go b/crypto/util_test.go index 667da0bcd0..260dd18fca 100644 --- a/crypto/util_test.go +++ b/crypto/util_test.go @@ -17,8 +17,10 @@ package crypto import ( + "fmt" "testing" + "github.com/algorand/go-algorand/protocol" "github.com/algorand/go-algorand/test/partitiontest" "github.com/stretchr/testify/require" ) @@ -46,3 +48,31 @@ func TestDigest_IsZero(t *testing.T) { require.NotZero(t, d2) } + +type testToBeHashed struct { + i int +} + +func (tbh *testToBeHashed) ToBeHashed() (protocol.HashID, []byte) { + data := make([]byte, tbh.i) + for x := 0; x < tbh.i; x++ { + data[x] = byte(tbh.i) + } + return protocol.HashID(fmt.Sprintf("ID%d", tbh.i)), data +} + +func TestHashRepToBuff(t *testing.T) { + values := []int{32, 64, 512, 1024} + buffer := make([]byte, 0, 128) + for _, val := range values { + tbh := &testToBeHashed{i: val} + buffer = HashRepToBuff(tbh, buffer) + } + pos := 0 + for _, val := range values { + tbh := &testToBeHashed{i: val} + data := HashRep(tbh) + require.Equal(t, data, buffer[pos:pos+len(data)]) + pos = pos + len(data) + } +} From 56a5154b15de1cf79e804d73b709638852bf4825 Mon Sep 17 00:00:00 2001 From: Shant Karakashian <55754073+algonautshant@users.noreply.github.com> Date: Mon, 28 Aug 2023 21:45:37 -0400 Subject: [PATCH 06/11] CR: suggested make it more readable Co-authored-by: Pavel Zbitskiy <65323360+algorandskiy@users.noreply.github.com> --- crypto/onetimesig.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crypto/onetimesig.go b/crypto/onetimesig.go index c4f9fef5cd..393afd2e4f 100644 --- a/crypto/onetimesig.go +++ b/crypto/onetimesig.go @@ -319,15 +319,18 @@ func (v OneTimeSignatureVerifier) Verify(id OneTimeSignatureIdentifier, message Batch: id.Batch, } - msgLengths := make([]int, 0, 3) - estimatedSize := 256 + // serialize encoded batchID, offsetID, message into a continuous memory buffer with the layout + // hashRep(batchID)... hashRep(offsetID)... hashRep(message)... + const estimatedSize = 256 messageBuffer := make([]byte, 0, estimatedSize) + messageBuffer = HashRepToBuff(batchID, messageBuffer) - msgLengths = append(msgLengths, len(messageBuffer)) + batchIDLen = len(messageBuffer) messageBuffer = HashRepToBuff(offsetID, messageBuffer) - msgLengths = append(msgLengths, len(messageBuffer)-msgLengths[0]) + offsetIDLen = len(messageBuffer) - batchIDLen messageBuffer = HashRepToBuff(message, messageBuffer) - msgLengths = append(msgLengths, len(messageBuffer)-msgLengths[1]-msgLengths[0]) + messageLen = len(messageBuffer) - offsetIDLen - batchIDLen + msgLengths := []int{batchIDLen, offsetIDLen, messageLen} allValid, _ := batchVerificationImpl( messageBuffer, msgLengths, From 4f0483947b2d196a7c8373a77597cf0401eb305e Mon Sep 17 00:00:00 2001 From: algonautshant Date: Mon, 28 Aug 2023 23:41:13 -0400 Subject: [PATCH 07/11] fix --- crypto/onetimesig.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crypto/onetimesig.go b/crypto/onetimesig.go index 393afd2e4f..9b76b453a0 100644 --- a/crypto/onetimesig.go +++ b/crypto/onetimesig.go @@ -325,11 +325,11 @@ func (v OneTimeSignatureVerifier) Verify(id OneTimeSignatureIdentifier, message messageBuffer := make([]byte, 0, estimatedSize) messageBuffer = HashRepToBuff(batchID, messageBuffer) - batchIDLen = len(messageBuffer) + batchIDLen := len(messageBuffer) messageBuffer = HashRepToBuff(offsetID, messageBuffer) - offsetIDLen = len(messageBuffer) - batchIDLen + offsetIDLen := len(messageBuffer) - batchIDLen messageBuffer = HashRepToBuff(message, messageBuffer) - messageLen = len(messageBuffer) - offsetIDLen - batchIDLen + messageLen := len(messageBuffer) - offsetIDLen - batchIDLen msgLengths := []int{batchIDLen, offsetIDLen, messageLen} allValid, _ := batchVerificationImpl( messageBuffer, From a95de53fbf55f2740ee74e9a2817ef79cd271007 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 29 Aug 2023 00:58:58 -0400 Subject: [PATCH 08/11] add partition to test --- crypto/util_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/crypto/util_test.go b/crypto/util_test.go index 260dd18fca..2e0828bcce 100644 --- a/crypto/util_test.go +++ b/crypto/util_test.go @@ -62,6 +62,7 @@ func (tbh *testToBeHashed) ToBeHashed() (protocol.HashID, []byte) { } func TestHashRepToBuff(t *testing.T) { + partitiontest.PartitionTest(t) values := []int{32, 64, 512, 1024} buffer := make([]byte, 0, 128) for _, val := range values { From 607da07b0ef4df636509d7f17303dfd35815bee1 Mon Sep 17 00:00:00 2001 From: algonautshant Date: Tue, 29 Aug 2023 12:20:25 -0400 Subject: [PATCH 09/11] CR: do not copy mesg lengths to C.ulonglong --- crypto/batchverifier.go | 18 ++++++++---------- crypto/onetimesig.go | 8 ++++---- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index eb0ffce20c..c5d5a062f6 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -128,13 +128,15 @@ func (b *BatchVerifier) VerifyWithFeedback() (failed []bool, err error) { if b.GetNumberOfEnqueuedSignatures() == 0 { return nil, nil } - msgLengths := make([]int, 0, len(b.messages)) - var messages = make([]byte, 0, b.GetNumberOfEnqueuedSignatures()*64) + + const estimatedMessageSize = 64 + msgLengths := make([]uint64, 0, len(b.messages)) + var messages = make([]byte, 0, b.GetNumberOfEnqueuedSignatures()*estimatedMessageSize) lenWas := 0 for i := range b.messages { messages = HashRepToBuff(b.messages[i], messages) - msgLengths = append(msgLengths, len(messages)-lenWas) + msgLengths = append(msgLengths, uint64(len(messages)-lenWas)) lenWas = len(messages) } allValid, failed := batchVerificationImpl(messages, msgLengths, b.publicKeys, b.signatures) @@ -147,19 +149,15 @@ func (b *BatchVerifier) VerifyWithFeedback() (failed []bool, err error) { // batchVerificationImpl invokes the ed25519 batch verification algorithm. // it returns true if all the signatures were authentically signed by the owners // otherwise, returns false, and sets the indexes of the failed sigs in failed -func batchVerificationImpl(messages []byte, msgLengths []int, publicKeys []SignatureVerifier, signatures []Signature) (allSigsValid bool, failed []bool) { +func batchVerificationImpl(messages []byte, msgLengths []uint64, publicKeys []SignatureVerifier, signatures []Signature) (allSigsValid bool, failed []bool) { numberOfSignatures := len(msgLengths) - messageLens := make([]C.ulonglong, len(msgLengths)) - for i, l := range msgLengths { - messageLens[i] = C.ulonglong(l) - } valid := make([]C.int, numberOfSignatures) // call the batch verifier allValid := C.ed25519_batch_wrapper( (*C.uchar)(&messages[0]), - (*C.ulonglong)(&messageLens[0]), + (*C.ulonglong)(&msgLengths[0]), (*C.uchar)(&(publicKeys[0][0])), (*C.uchar)(&(signatures[0][0])), C.size_t(numberOfSignatures), @@ -167,7 +165,7 @@ func batchVerificationImpl(messages []byte, msgLengths []int, publicKeys []Signa // These calls will be replaced with Pin in the next Go version upgrade runtime.KeepAlive(messages) - runtime.KeepAlive(messageLens) + runtime.KeepAlive(msgLengths) runtime.KeepAlive(publicKeys) runtime.KeepAlive(signatures) diff --git a/crypto/onetimesig.go b/crypto/onetimesig.go index 9b76b453a0..b42eb31cec 100644 --- a/crypto/onetimesig.go +++ b/crypto/onetimesig.go @@ -325,12 +325,12 @@ func (v OneTimeSignatureVerifier) Verify(id OneTimeSignatureIdentifier, message messageBuffer := make([]byte, 0, estimatedSize) messageBuffer = HashRepToBuff(batchID, messageBuffer) - batchIDLen := len(messageBuffer) + batchIDLen := uint64(len(messageBuffer)) messageBuffer = HashRepToBuff(offsetID, messageBuffer) - offsetIDLen := len(messageBuffer) - batchIDLen + offsetIDLen := uint64(len(messageBuffer)) - batchIDLen messageBuffer = HashRepToBuff(message, messageBuffer) - messageLen := len(messageBuffer) - offsetIDLen - batchIDLen - msgLengths := []int{batchIDLen, offsetIDLen, messageLen} + messageLen := uint64(len(messageBuffer)) - offsetIDLen - batchIDLen + msgLengths := []uint64{batchIDLen, offsetIDLen, messageLen} allValid, _ := batchVerificationImpl( messageBuffer, msgLengths, From 2dde9bc4f7ecc3bafb445d6cd8a513d34eaf7650 Mon Sep 17 00:00:00 2001 From: chris erway Date: Fri, 17 Nov 2023 12:14:43 -0500 Subject: [PATCH 10/11] CR update and slice allocation runtime check for #5700 --- crypto/batchverifier.go | 7 ------- crypto/curve25519.go | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index c5d5a062f6..ab58822419 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -43,7 +43,6 @@ package crypto import "C" import ( "errors" - "runtime" "unsafe" ) @@ -163,12 +162,6 @@ func batchVerificationImpl(messages []byte, msgLengths []uint64, publicKeys []Si C.size_t(numberOfSignatures), (*C.int)(&valid[0])) - // These calls will be replaced with Pin in the next Go version upgrade - runtime.KeepAlive(messages) - runtime.KeepAlive(msgLengths) - runtime.KeepAlive(publicKeys) - runtime.KeepAlive(signatures) - failed = make([]bool, numberOfSignatures) for i := 0; i < numberOfSignatures; i++ { failed[i] = (valid[i] == 0) diff --git a/crypto/curve25519.go b/crypto/curve25519.go index 58950a3de3..a8637399d7 100644 --- a/crypto/curve25519.go +++ b/crypto/curve25519.go @@ -35,6 +35,7 @@ import "C" import ( "fmt" + "unsafe" "github.com/algorand/go-algorand/logging" "github.com/algorand/go-algorand/util/metrics" @@ -64,6 +65,30 @@ func init() { _ = [C.crypto_sign_ed25519_PUBLICKEYBYTES]byte(ed25519PublicKey{}) _ = [C.crypto_sign_ed25519_SECRETKEYBYTES]byte(ed25519PrivateKey{}) _ = [C.crypto_sign_ed25519_SEEDBYTES]byte(ed25519Seed{}) + + // Check that this platform makes slices []Signature and []SignatureVerifier that use a backing + // array of contiguously allocated 64- and 32-byte segments, respectively, with no padding. + // These slice's backing arrays are passed to C.ed25519_batch_wrapper. In practice, this check + // should always succeed, but to be careful we can double-check, since the Go specification does + // not explicitly define platform-specific alignment sizes and slice allocation behavior. + length := 1024 + sigs := make([]Signature, length) // same as [][64]byte + pks := make([]SignatureVerifier, length) // same as [][32]byte + + for i := 1; i < length; i++ { + if uintptr(unsafe.Pointer(&sigs[i]))-uintptr(unsafe.Pointer(&sigs[0])) != uintptr(i)*C.crypto_sign_ed25519_BYTES { + panic("Unexpected alignment for a slice of signatures") + } + if uintptr(unsafe.Pointer(&pks[i]))-uintptr(unsafe.Pointer(&pks[0])) != uintptr(i)*C.crypto_sign_ed25519_PUBLICKEYBYTES { + panic("Unexpected alignment for a slice of public keys") + } + } + if uintptr(unsafe.Pointer(&sigs[length-1]))-uintptr(unsafe.Pointer(&sigs[0])) != uintptr(length-1)*C.crypto_sign_ed25519_BYTES { + panic("Unexpected total size for a backing array of signatures") + } + if uintptr(unsafe.Pointer(&pks[length-1]))-uintptr(unsafe.Pointer(&pks[0])) != uintptr(length-1)*C.crypto_sign_ed25519_PUBLICKEYBYTES { + panic("Unexpected total size for a backing array of public keys") + } } // A Seed holds the entropy needed to generate cryptographic keys. From a50dc52fa5557d8066c809169f63af66d9560f24 Mon Sep 17 00:00:00 2001 From: chris erway Date: Fri, 17 Nov 2023 12:53:17 -0500 Subject: [PATCH 11/11] allocate space for 2D arrays in Go, rather than C --- crypto/batchverifier.c | 18 +++++------------- crypto/batchverifier.go | 17 ++++++++++++----- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/crypto/batchverifier.c b/crypto/batchverifier.c index 8fd233ab8f..118542aa7b 100644 --- a/crypto/batchverifier.c +++ b/crypto/batchverifier.c @@ -1,16 +1,13 @@ #include "sodium.h" -int ed25519_batch_wrapper(const unsigned char *messages1D, +int ed25519_batch_wrapper(const unsigned char **messages2D, + const unsigned char **publicKeys2D, + const unsigned char **signatures2D, + const unsigned char *messages1D, const unsigned long long *mlen, const unsigned char *publicKeys1D, const unsigned char *signatures1D, size_t num, int *valid) { - int ret; - const unsigned char **messages2D, **publicKeys2D, **signatures2D; - messages2D = malloc(num * sizeof(unsigned char *)); - publicKeys2D = malloc(num * sizeof(unsigned char *)); - signatures2D = malloc(num * sizeof(unsigned char *)); - // fill 2-D arrays for messages, pks, sigs from provided 1-D arrays unsigned long long mpos = 0; for (size_t i = 0; i < num; i++) { @@ -19,10 +16,5 @@ int ed25519_batch_wrapper(const unsigned char *messages1D, publicKeys2D[i] = &publicKeys1D[i*crypto_sign_ed25519_PUBLICKEYBYTES]; signatures2D[i] = &signatures1D[i*crypto_sign_ed25519_BYTES]; } - ret = crypto_sign_ed25519_open_batch(messages2D, mlen, publicKeys2D, signatures2D, num, valid); - - free(messages2D); - free(publicKeys2D); - free(signatures2D); - return ret; + return crypto_sign_ed25519_open_batch(messages2D, mlen, publicKeys2D, signatures2D, num, valid); } diff --git a/crypto/batchverifier.go b/crypto/batchverifier.go index ab58822419..af7a677ac3 100644 --- a/crypto/batchverifier.go +++ b/crypto/batchverifier.go @@ -34,7 +34,10 @@ package crypto // sizeofPtr = sizeof(void*), // sizeofULongLong = sizeof(unsigned long long), // }; -// int ed25519_batch_wrapper(const unsigned char *messages1D, +// int ed25519_batch_wrapper(const unsigned char **messages2D, +// const unsigned char **publicKeys2D, +// const unsigned char **signatures2D, +// const unsigned char *messages1D, // const unsigned long long *mlen, // const unsigned char *publicKeys1D, // const unsigned char *signatures1D, @@ -124,13 +127,13 @@ func (b *BatchVerifier) Verify() error { // if some signatures are invalid, true will be set in failed at the corresponding indexes, and // ErrBatchVerificationFailed for err func (b *BatchVerifier) VerifyWithFeedback() (failed []bool, err error) { - if b.GetNumberOfEnqueuedSignatures() == 0 { + if len(b.messages) == 0 { return nil, nil } const estimatedMessageSize = 64 msgLengths := make([]uint64, 0, len(b.messages)) - var messages = make([]byte, 0, b.GetNumberOfEnqueuedSignatures()*estimatedMessageSize) + var messages = make([]byte, 0, len(b.messages)*estimatedMessageSize) lenWas := 0 for i := range b.messages { @@ -152,13 +155,17 @@ func batchVerificationImpl(messages []byte, msgLengths []uint64, publicKeys []Si numberOfSignatures := len(msgLengths) valid := make([]C.int, numberOfSignatures) + messages2D := make([]*C.uchar, numberOfSignatures) + publicKeys2D := make([]*C.uchar, numberOfSignatures) + signatures2D := make([]*C.uchar, numberOfSignatures) // call the batch verifier allValid := C.ed25519_batch_wrapper( + &messages2D[0], &publicKeys2D[0], &signatures2D[0], (*C.uchar)(&messages[0]), (*C.ulonglong)(&msgLengths[0]), - (*C.uchar)(&(publicKeys[0][0])), - (*C.uchar)(&(signatures[0][0])), + (*C.uchar)(&publicKeys[0][0]), + (*C.uchar)(&signatures[0][0]), C.size_t(numberOfSignatures), (*C.int)(&valid[0]))