batch verification: add batch verification implementation (#3031)

## Summary
In this change, we plan to integrate the ed25519  batch verification algorithm. This will speed up the process of verifying the digital signature of several transactions by roughly 2.4X.

This PR implements the algorithm according to the following paper:
https://eprint.iacr.org/2020/1244.pdf
This PR fixes a **bug** in the paper (Listing 1.2) that didn't reject the points (-0,-1) and (-0,1)

Another blog article with similar background: https://hdevalence.ca/blog/2020-10-04-its-25519am

**Changes Overview**
Changes made on the single ed255519 verification :
1. Signatures that contain small order group R value will now be accepted.
2. use the cofactor version of the equation to validate the signature. (multiple the result by the cofactor and compare to natural element)
3. add ge25519_is_canonical_vartime function (check if a given y point has a canonical or non-canonical encoding  ): 
    - optimize this function using the the quick check.
    - reject two non-canonical points by comparison (points (-0,-1) and (-0,1), this resolves a bug in the paper). 

Add the batch verification implementation from https://github.com/floodyberry/ed25519-donna to our fork of libsodium.
Changes made on the batch verification (to match the single verification):
1. reject small order A (PK)
2. reject non-canonical A (PK)
3. reject non-canonical R
4. reject non-canonical S
5. use the cofactor equation 

## Test Plan

The following tests were added to the libsodium's test suite.
1. Test against the White Paper's test vector
2. Test against all non-canonical options for R
3. Test against all non-canonical options for S

Author: id-ms

agreement/vote.go

@@ -129,7 +129,7 @@ func (uv unauthenticatedVote) verify(l LedgerReader) (vote, error) {
 
 	ephID := basics.OneTimeIDForRound(rv.Round, m.Record.KeyDilution(proto))
 	voteID := m.Record.VoteID
-	if !voteID.Verify(ephID, rv, uv.Sig) {
+	if !voteID.Verify(ephID, rv, uv.Sig, proto.EnableBatchVerification) {
 		return vote{}, fmt.Errorf("unauthenticatedVote.verify: could not verify FS signature on vote by %v given %v: %+v", rv.Sender, voteID, uv)
 	}
 

config/consensus.go

@@ -395,6 +395,9 @@ type ConsensusParams struct {
 	// to be taken offline, that would be proposed to be taken offline.
 	MaxProposedExpiredOnlineAccounts int
 
+	//EnableBatchVerification enable the use of the batch verification algorithm.
+	EnableBatchVerification bool
+
 	// When rewards rate changes, use the new value immediately.
 	RewardsCalculationFix bool
 }
@@ -1057,6 +1060,8 @@ func initConsensusProtocols() {
 
 	vFuture.MaxProposedExpiredOnlineAccounts = 32
 
+	vFuture.EnableBatchVerification = true
+
 	vFuture.RewardsCalculationFix = true
 
 	Consensus[protocol.ConsensusFuture] = vFuture

crypto/batchverifier.go

@@ -16,40 +16,78 @@
 
 package crypto
 
-import "errors"
+// #cgo CFLAGS: -Wall -std=c99
+// #cgo darwin,amd64 CFLAGS: -I${SRCDIR}/libs/darwin/amd64/include
+// #cgo darwin,amd64 LDFLAGS: ${SRCDIR}/libs/darwin/amd64/lib/libsodium.a
+// #cgo darwin,arm64 CFLAGS: -I${SRCDIR}/libs/darwin/arm64/include
+// #cgo darwin,arm64 LDFLAGS: ${SRCDIR}/libs/darwin/arm64/lib/libsodium.a
+// #cgo linux,amd64 CFLAGS: -I${SRCDIR}/libs/linux/amd64/include
+// #cgo linux,amd64 LDFLAGS: ${SRCDIR}/libs/linux/amd64/lib/libsodium.a
+// #cgo linux,arm64 CFLAGS: -I${SRCDIR}/libs/linux/arm64/include
+// #cgo linux,arm64 LDFLAGS: ${SRCDIR}/libs/linux/arm64/lib/libsodium.a
+// #cgo linux,arm CFLAGS: -I${SRCDIR}/libs/linux/arm/include
+// #cgo linux,arm LDFLAGS: ${SRCDIR}/libs/linux/arm/lib/libsodium.a
+// #cgo windows,amd64 CFLAGS: -I${SRCDIR}/libs/windows/amd64/include
+// #cgo windows,amd64 LDFLAGS: ${SRCDIR}/libs/windows/amd64/lib/libsodium.a
+// #include <stdint.h>
+// #include "sodium.h"
+// enum {
+//	sizeofPtr = sizeof(void*),
+//	sizeofULongLong = sizeof(unsigned long long),
+// };
+import "C"
+import (
+	"errors"
+	"unsafe"
+)
 
 // BatchVerifier enqueues signatures to be validated in batch.
 type BatchVerifier struct {
-	messages   []Hashable          // contains a slice of messages to be hashed. Each message is varible length
-	publicKeys []SignatureVerifier // contains a slice of public keys. Each individual public key is 32 bytes.
-	signatures []Signature         // contains a slice of signatures keys. Each individual signature is 64 bytes.
+	messages             []Hashable          // contains a slice of messages to be hashed. Each message is varible length
+	publicKeys           []SignatureVerifier // contains a slice of public keys. Each individual public key is 32 bytes.
+	signatures           []Signature         // contains a slice of signatures keys. Each individual signature is 64 bytes.
+	useBatchVerification bool
 }
 
 const minBatchVerifierAlloc = 16
 
 // Batch verifications errors
 var (
 	ErrBatchVerificationFailed = errors.New("At least one signature didn't pass verification")
-	ErrZeroTranscationsInBatch = errors.New("Could not validate empty signature set")
+	ErrZeroTransactionInBatch  = errors.New("Could not validate empty signature set")
 )
 
+//export ed25519_randombytes_unsafe
+func ed25519_randombytes_unsafe(p unsafe.Pointer, len C.size_t) {
+	randBuf := (*[1 << 30]byte)(p)[:len:len]
+	RandBytes(randBuf)
+}
+
+// MakeBatchVerifierWithAlgorithmDefaultSize create a BatchVerifier instance. This function pre-allocates
+// amount of free space to enqueue signatures without expanding. this function always use the batch
+// verification algorithm
+func MakeBatchVerifierWithAlgorithmDefaultSize() *BatchVerifier {
+	return MakeBatchVerifier(minBatchVerifierAlloc, true)
+}
+
 // MakeBatchVerifierDefaultSize create a BatchVerifier instance. This function pre-allocates
-// amount of free space to enqueue signatures without exapneding
-func MakeBatchVerifierDefaultSize() *BatchVerifier {
-	return MakeBatchVerifier(minBatchVerifierAlloc)
+// amount of free space to enqueue signatures without expanding
+func MakeBatchVerifierDefaultSize(enableBatchVerification bool) *BatchVerifier {
+	return MakeBatchVerifier(minBatchVerifierAlloc, enableBatchVerification)
 }
 
 // MakeBatchVerifier create a BatchVerifier instance. This function pre-allocates
 // a given space so it will not expaned the storage
-func MakeBatchVerifier(hint int) *BatchVerifier {
+func MakeBatchVerifier(hint int, enableBatchVerification bool) *BatchVerifier {
 	// preallocate enough storage for the expected usage. We will reallocate as needed.
 	if hint < minBatchVerifierAlloc {
 		hint = minBatchVerifierAlloc
 	}
 	return &BatchVerifier{
-		messages:   make([]Hashable, 0, hint),
-		publicKeys: make([]SignatureVerifier, 0, hint),
-		signatures: make([]Signature, 0, hint),
+		messages:             make([]Hashable, 0, hint),
+		publicKeys:           make([]SignatureVerifier, 0, hint),
+		signatures:           make([]Signature, 0, hint),
+		useBatchVerification: enableBatchVerification,
 	}
 }
 
@@ -85,14 +123,69 @@ func (b *BatchVerifier) GetNumberOfEnqueuedSignatures() int {
 // if the batch is zero an appropriate error is return.
 func (b *BatchVerifier) Verify() error {
 	if b.GetNumberOfEnqueuedSignatures() == 0 {
-		return ErrZeroTranscationsInBatch
+		return ErrZeroTransactionInBatch
 	}
 
+	if b.useBatchVerification {
+		var messages = make([][]byte, b.GetNumberOfEnqueuedSignatures())
+		for i, m := range b.messages {
+			messages[i] = hashRep(m)
+		}
+		if batchVerificationImpl(messages, b.publicKeys, b.signatures) {
+			return nil
+		}
+		return ErrBatchVerificationFailed
+	}
+	return b.verifyOneByOne()
+}
+
+func (b *BatchVerifier) verifyOneByOne() error {
 	for i := range b.messages {
-		verifier := SignatureVerifier(b.publicKeys[i])
-		if !verifier.Verify(b.messages[i], b.signatures[i]) {
+		verifier := b.publicKeys[i]
+		if !verifier.Verify(b.messages[i], b.signatures[i], false) {
 			return ErrBatchVerificationFailed
 		}
 	}
 	return nil
 }
+
+// batchVerificationImpl invokes the ed25519 batch verification algorithm.
+// it returns true if all the signatures were authentically signed by the owners
+func batchVerificationImpl(messages [][]byte, publicKeys []SignatureVerifier, signatures []Signature) bool {
+
+	numberOfSignatures := len(messages)
+
+	messagesAllocation := C.malloc(C.size_t(C.sizeofPtr * numberOfSignatures))
+	messagesLenAllocation := C.malloc(C.size_t(C.sizeofULongLong * numberOfSignatures))
+	publicKeysAllocation := C.malloc(C.size_t(C.sizeofPtr * numberOfSignatures))
+	signaturesAllocation := C.malloc(C.size_t(C.sizeofPtr * numberOfSignatures))
+	valid := C.malloc(C.size_t(C.sizeof_int * numberOfSignatures))
+
+	defer func() {
+		// release staging memory
+		C.free(messagesAllocation)
+		C.free(messagesLenAllocation)
+		C.free(publicKeysAllocation)
+		C.free(signaturesAllocation)
+		C.free(valid)
+	}()
+
+	// load all the data pointers into the array pointers.
+	for i := 0; i < numberOfSignatures; i++ {
+		*(*uintptr)(unsafe.Pointer(uintptr(messagesAllocation) + uintptr(i*C.sizeofPtr))) = uintptr(unsafe.Pointer(&messages[i][0]))
+		*(*C.ulonglong)(unsafe.Pointer(uintptr(messagesLenAllocation) + uintptr(i*C.sizeofULongLong))) = C.ulonglong(len(messages[i]))
+		*(*uintptr)(unsafe.Pointer(uintptr(publicKeysAllocation) + uintptr(i*C.sizeofPtr))) = uintptr(unsafe.Pointer(&publicKeys[i][0]))
+		*(*uintptr)(unsafe.Pointer(uintptr(signaturesAllocation) + uintptr(i*C.sizeofPtr))) = uintptr(unsafe.Pointer(&signatures[i][0]))
+	}
+
+	// call the batch verifier
+	allValid := C.crypto_sign_ed25519_open_batch(
+		(**C.uchar)(unsafe.Pointer(messagesAllocation)),
+		(*C.ulonglong)(unsafe.Pointer(messagesLenAllocation)),
+		(**C.uchar)(unsafe.Pointer(publicKeysAllocation)),
+		(**C.uchar)(unsafe.Pointer(signaturesAllocation)),
+		C.size_t(len(messages)),
+		(*C.int)(unsafe.Pointer(valid)))
+
+	return allValid == 0
+}

crypto/batchverifier_test.go

@@ -27,7 +27,7 @@ import (
 func TestBatchVerifierSingle(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	// test expected success
-	bv := MakeBatchVerifier(1)
+	bv := MakeBatchVerifierWithAlgorithmDefaultSize()
 	msg := randString()
 	var s Seed
 	RandBytes(s[:])
@@ -36,8 +36,8 @@ func TestBatchVerifierSingle(t *testing.T) {
 	bv.EnqueueSignature(sigSecrets.SignatureVerifier, msg, sig)
 	require.NoError(t, bv.Verify())
 
-	// test expected failuire
-	bv = MakeBatchVerifier(1)
+	// test expected failure
+	bv = MakeBatchVerifierWithAlgorithmDefaultSize()
 	msg = randString()
 	RandBytes(s[:])
 	sigSecrets = GenerateSignatureSecrets(s)
@@ -52,7 +52,7 @@ func TestBatchVerifierBulk(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	for i := 1; i < 64*2+3; i++ {
 		n := i
-		bv := MakeBatchVerifier(n)
+		bv := MakeBatchVerifier(n, true)
 		var s Seed
 
 		for i := 0; i < n; i++ {
@@ -71,7 +71,7 @@ func TestBatchVerifierBulk(t *testing.T) {
 func TestBatchVerifierBulkWithExpand(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	n := 64
-	bv := MakeBatchVerifier(1)
+	bv := MakeBatchVerifierWithAlgorithmDefaultSize()
 	var s Seed
 	RandBytes(s[:])
 
@@ -87,7 +87,7 @@ func TestBatchVerifierBulkWithExpand(t *testing.T) {
 func TestBatchVerifierWithInvalidSiganture(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	n := 64
-	bv := MakeBatchVerifier(1)
+	bv := MakeBatchVerifierWithAlgorithmDefaultSize()
 	var s Seed
 	RandBytes(s[:])
 
@@ -109,7 +109,7 @@ func TestBatchVerifierWithInvalidSiganture(t *testing.T) {
 
 func BenchmarkBatchVerifier(b *testing.B) {
 	c := makeCurve25519Secret()
-	bv := MakeBatchVerifier(1)
+	bv := MakeBatchVerifier(1, true)
 	for i := 0; i < b.N; i++ {
 		str := randString()
 		bv.EnqueueSignature(c.SignatureVerifier, str, c.Sign(str))
@@ -121,6 +121,6 @@ func BenchmarkBatchVerifier(b *testing.B) {
 
 func TestEmpty(t *testing.T) {
 	partitiontest.PartitionTest(t)
-	bv := MakeBatchVerifierDefaultSize()
+	bv := MakeBatchVerifierWithAlgorithmDefaultSize()
 	require.Error(t, bv.Verify())
 }

crypto/compactcert/builder.go

@@ -99,7 +99,7 @@ func (b *Builder) Add(pos uint64, sig crypto.OneTimeSignature, verifySig bool) e
 
 	// Check signature
 	ephID := basics.OneTimeIDForRound(b.SigRound, p.KeyDilution)
-	if verifySig && !p.PK.Verify(ephID, b.Msg, sig) {
+	if verifySig && !p.PK.Verify(ephID, b.Msg, sig, b.EnableBatchVerification) {
 		return fmt.Errorf("signature does not verify under ID %v", ephID)
 	}
 

crypto/compactcert/structs.go

@@ -28,6 +28,8 @@ type Params struct {
 	ProvenWeight uint64          // Weight threshold proven by the certificate
 	SigRound     basics.Round    // Ephemeral signature round to expect
 	SecKQ        uint64          // Security parameter (k+q) from analysis document
+
+	EnableBatchVerification bool // whether ED25519 batch verification is enabled
 }
 
 // CompactOneTimeSignature is crypto.OneTimeSignature with omitempty

crypto/compactcert/verifier.go

@@ -56,7 +56,7 @@ func (v *Verifier) Verify(c *Cert) error {
 		parts[pos] = crypto.HashObj(r.Part)
 
 		ephID := basics.OneTimeIDForRound(v.SigRound, r.Part.KeyDilution)
-		if !r.Part.PK.Verify(ephID, v.Msg, r.SigSlot.Sig.OneTimeSignature) {
+		if !r.Part.PK.Verify(ephID, v.Msg, r.SigSlot.Sig.OneTimeSignature, v.EnableBatchVerification) {
 			return fmt.Errorf("signature in reveal pos %d does not verify", pos)
 		}
 	}

crypto/crypto_test.go

@@ -43,22 +43,22 @@ func randString() (b TestingHashable) {
 func signVerify(t *testing.T, c *SignatureSecrets, c2 *SignatureSecrets) {
 	s := randString()
 	sig := c.Sign(s)
-	if !c.Verify(s, sig) {
+	if !c.Verify(s, sig, true) {
 		t.Errorf("correct signature failed to verify (plain)")
 	}
 
 	s2 := randString()
 	sig2 := c.Sign(s2)
-	if c.Verify(s, sig2) {
+	if c.Verify(s, sig2, true) {
 		t.Errorf("wrong message incorrectly verified (plain)")
 	}
 
 	sig3 := c2.Sign(s)
-	if c.Verify(s, sig3) {
+	if c.Verify(s, sig3, true) {
 		t.Errorf("wrong key incorrectly verified (plain)")
 	}
 
-	if c.Verify(s2, sig3) {
+	if c.Verify(s2, sig3, true) {
 		t.Errorf("wrong message+key incorrectly verified (plain)")
 	}
 }

crypto/curve25519.go

@@ -112,14 +112,19 @@ func ed25519Sign(secret ed25519PrivateKey, data []byte) (sig ed25519Signature) {
 	return
 }
 
-func ed25519Verify(public ed25519PublicKey, data []byte, sig ed25519Signature) bool {
+func ed25519Verify(public ed25519PublicKey, data []byte, sig ed25519Signature, useBatchVerificationCompatibleVersion bool) bool {
 	// &data[0] will make Go panic if msg is zero length
 	d := (*C.uchar)(C.NULL)
 	if len(data) != 0 {
 		d = (*C.uchar)(&data[0])
 	}
 	// https://download.libsodium.org/doc/public-key_cryptography/public-key_signatures#detached-mode
-	result := C.crypto_sign_ed25519_verify_detached((*C.uchar)(&sig[0]), d, C.ulonglong(len(data)), (*C.uchar)(&public[0]))
+	var result C.int
+	if useBatchVerificationCompatibleVersion {
+		result = C.crypto_sign_ed25519_bv_compatible_verify_detached((*C.uchar)(&sig[0]), d, C.ulonglong(len(data)), (*C.uchar)(&public[0]))
+	} else {
+		result = C.crypto_sign_ed25519_verify_detached((*C.uchar)(&sig[0]), d, C.ulonglong(len(data)), (*C.uchar)(&public[0]))
+	}
 	return result == 0
 }
 
@@ -211,15 +216,15 @@ func (s *SignatureSecrets) SignBytes(message []byte) Signature {
 //
 // It returns true if this is the case; otherwise, it returns false.
 //
-func (v SignatureVerifier) Verify(message Hashable, sig Signature) bool {
+func (v SignatureVerifier) Verify(message Hashable, sig Signature, useBatchVerificationCompatibleVersion bool) bool {
 	cryptoSigSecretsVerifyTotal.Inc(map[string]string{})
-	return ed25519Verify(ed25519PublicKey(v), hashRep(message), ed25519Signature(sig))
+	return ed25519Verify(ed25519PublicKey(v), hashRep(message), ed25519Signature(sig), useBatchVerificationCompatibleVersion)
 }
 
 // VerifyBytes verifies a signature, where the message is not hashed first.
 // Caller is responsible for domain separation.
 // If the message is a Hashable, Verify() can be used instead.
-func (v SignatureVerifier) VerifyBytes(message []byte, sig Signature) bool {
+func (v SignatureVerifier) VerifyBytes(message []byte, sig Signature, useBatchVerificationCompatibleVersion bool) bool {
 	cryptoSigSecretsVerifyBytesTotal.Inc(map[string]string{})
-	return ed25519Verify(ed25519PublicKey(v), message, ed25519Signature(sig))
+	return ed25519Verify(ed25519PublicKey(v), message, ed25519Signature(sig), useBatchVerificationCompatibleVersion)
 }

crypto/curve25519_test.go

@@ -33,7 +33,7 @@ func TestSignVerifyEmptyMessage(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	pk, sk := ed25519GenerateKey()
 	sig := ed25519Sign(sk, []byte{})
-	if !ed25519Verify(pk, []byte{}, sig) {
+	if !ed25519Verify(pk, []byte{}, sig, true) {
 		t.Errorf("sig of an empty message failed to verify")
 	}
 }
@@ -43,7 +43,7 @@ func TestVerifyZeros(t *testing.T) {
 	var pk SignatureVerifier
 	var sig Signature
 	for x := byte(0); x < 255; x++ {
-		if pk.VerifyBytes([]byte{x}, sig) {
+		if pk.VerifyBytes([]byte{x}, sig, true) {
 			t.Errorf("Zero sig with zero pk successfully verified message %x", x)
 		}
 	}
@@ -84,7 +84,7 @@ func BenchmarkSignVerify(b *testing.B) {
 
 	for i := 0; i < b.N; i++ {
 		sig := c.Sign(s)
-		_ = c.Verify(s, sig)
+		_ = c.Verify(s, sig, true)
 	}
 }
 
@@ -108,6 +108,6 @@ func BenchmarkVerify(b *testing.B) {
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
-		_ = c.Verify(strs[i], sigs[i])
+		_ = c.Verify(strs[i], sigs[i], true)
 	}
 }