Merge PR: replace ics23 (#2584)

Co-authored-by: KamiD <[email protected]>

go.mod

@@ -15,7 +15,6 @@ require (
 	github.com/bgentry/speakeasy v0.1.0
 	github.com/btcsuite/btcd v0.21.0-beta
 	github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce
-	github.com/confio/ics23/go v0.7.0
 	github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d
 	github.com/cosmos/gorocksdb v1.2.0
 	github.com/cosmos/ledger-cosmos-go v0.11.1

go.sum

@@ -167,8 +167,6 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
-github.com/confio/ics23/go v0.7.0 h1:00d2kukk7sPoHWL4zZBZwzxnpA2pec1NPdwbSokJ5w8=
-github.com/confio/ics23/go v0.7.0/go.mod h1:E45NqnlpxGnpfTWL/xauN7MRwEE28T4Dd4uraToOaKg=
 github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ=
 github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=

ics23/Makefile

@@ -0,0 +1,20 @@
+.PHONY: protoc test
+
+# make sure we turn on go modules
+export GO111MODULE := on
+
+# PROTOC_FLAGS := -I=.. -I=./vendor -I=$(GOPATH)/src
+PROTOC_FLAGS := -I=.. -I=$(GOPATH)/src
+
+test:
+	go test .
+
+protoc:
+#	@go mod vendor
+	protoc --gocosmos_out=plugins=interfacetype+grpc,Mgoogle/protobuf/any.proto=github.com/gogo/protobuf/types:. $(PROTOC_FLAGS) ../proofs.proto
+
+install-proto-dep:
+	@echo "Installing protoc-gen-gocosmos..."
+	@go install github.com/regen-network/cosmos-proto/protoc-gen-gocosmos
+
+

ics23/compress.go

@@ -0,0 +1,157 @@
+package ics23
+
+// IsCompressed returns true if the proof was compressed
+func IsCompressed(proof *CommitmentProof) bool {
+	return proof.GetCompressed() != nil
+}
+
+// Compress will return a CompressedBatchProof if the input is BatchProof
+// Otherwise it will return the input.
+// This is safe to call multiple times (idempotent)
+func Compress(proof *CommitmentProof) *CommitmentProof {
+	batch := proof.GetBatch()
+	if batch == nil {
+		return proof
+	}
+	return &CommitmentProof{
+		Proof: &CommitmentProof_Compressed{
+			Compressed: compress(batch),
+		},
+	}
+}
+
+// Decompress will return a BatchProof if the input is CompressedBatchProof
+// Otherwise it will return the input.
+// This is safe to call multiple times (idempotent)
+func Decompress(proof *CommitmentProof) *CommitmentProof {
+	comp := proof.GetCompressed()
+	if comp != nil {
+		return &CommitmentProof{
+			Proof: &CommitmentProof_Batch{
+				Batch: decompress(comp),
+			},
+		}
+	}
+	return proof
+}
+
+func compress(batch *BatchProof) *CompressedBatchProof {
+	var centries []*CompressedBatchEntry
+	var lookup []*InnerOp
+	registry := make(map[string]int32)
+
+	for _, entry := range batch.Entries {
+		centry := compressEntry(entry, &lookup, registry)
+		centries = append(centries, centry)
+	}
+
+	return &CompressedBatchProof{
+		Entries:      centries,
+		LookupInners: lookup,
+	}
+}
+
+func compressEntry(entry *BatchEntry, lookup *[]*InnerOp, registry map[string]int32) *CompressedBatchEntry {
+	if exist := entry.GetExist(); exist != nil {
+		return &CompressedBatchEntry{
+			Proof: &CompressedBatchEntry_Exist{
+				Exist: compressExist(exist, lookup, registry),
+			},
+		}
+	}
+
+	non := entry.GetNonexist()
+	return &CompressedBatchEntry{
+		Proof: &CompressedBatchEntry_Nonexist{
+			Nonexist: &CompressedNonExistenceProof{
+				Key:   non.Key,
+				Left:  compressExist(non.Left, lookup, registry),
+				Right: compressExist(non.Right, lookup, registry),
+			},
+		},
+	}
+}
+
+func compressExist(exist *ExistenceProof, lookup *[]*InnerOp, registry map[string]int32) *CompressedExistenceProof {
+	if exist == nil {
+		return nil
+	}
+	res := &CompressedExistenceProof{
+		Key:   exist.Key,
+		Value: exist.Value,
+		Leaf:  exist.Leaf,
+		Path:  make([]int32, len(exist.Path)),
+	}
+	for i, step := range exist.Path {
+		res.Path[i] = compressStep(step, lookup, registry)
+	}
+	return res
+}
+
+func compressStep(step *InnerOp, lookup *[]*InnerOp, registry map[string]int32) int32 {
+	bz, err := step.Marshal()
+	if err != nil {
+		panic(err)
+	}
+	sig := string(bz)
+
+	// load from cache if there
+	if num, ok := registry[sig]; ok {
+		return num
+	}
+
+	// create new step if not there
+	num := int32(len(*lookup))
+	*lookup = append(*lookup, step)
+	registry[sig] = num
+	return num
+}
+
+func decompress(comp *CompressedBatchProof) *BatchProof {
+	lookup := comp.LookupInners
+
+	var entries []*BatchEntry
+
+	for _, centry := range comp.Entries {
+		entry := decompressEntry(centry, lookup)
+		entries = append(entries, entry)
+	}
+
+	return &BatchProof{
+		Entries: entries,
+	}
+}
+
+// TendermintSpec constrains the format from proofs-tendermint (crypto/merkle SimpleProof)
+var TendermintSpec = &ProofSpec{
+	LeafSpec: &LeafOp{
+		Prefix:       []byte{0},
+		PrehashKey:   HashOp_NO_HASH,
+		Hash:         HashOp_SHA256,
+		PrehashValue: HashOp_SHA256,
+		Length:       LengthOp_VAR_PROTO,
+	},
+	InnerSpec: &InnerSpec{
+		ChildOrder:      []int32{0, 1},
+		MinPrefixLength: 1,
+		MaxPrefixLength: 1,
+		ChildSize:       32, // (no length byte)
+		Hash:            HashOp_SHA256,
+	},
+}
+
+func decompressExist(exist *CompressedExistenceProof, lookup []*InnerOp) *ExistenceProof {
+	if exist == nil {
+		return nil
+	}
+	res := &ExistenceProof{
+		Key:   exist.Key,
+		Value: exist.Value,
+		Leaf:  exist.Leaf,
+		Path:  make([]*InnerOp, len(exist.Path)),
+	}
+	for i, step := range exist.Path {
+		res.Path[i] = lookup[step]
+	}
+	return res
+}

ics23/ics23.go

@@ -0,0 +1,174 @@
+/**
+This implements the client side functions as specified in
+https://github.com/cosmos/ics/tree/master/spec/ics-023-vector-commitments
+
+In particular:
+
+  // Assumes ExistenceProof
+  type verifyMembership = (root: CommitmentRoot, proof: CommitmentProof, key: Key, value: Value) => boolean
+
+  // Assumes NonExistenceProof
+  type verifyNonMembership = (root: CommitmentRoot, proof: CommitmentProof, key: Key) => boolean
+
+  // Assumes BatchProof - required ExistenceProofs may be a subset of all items proven
+  type batchVerifyMembership = (root: CommitmentRoot, proof: CommitmentProof, items: Map<Key, Value>) => boolean
+
+  // Assumes BatchProof - required NonExistenceProofs may be a subset of all items proven
+  type batchVerifyNonMembership = (root: CommitmentRoot, proof: CommitmentProof, keys: Set<Key>) => boolean
+
+We make an adjustment to accept a Spec to ensure the provided proof is in the format of the expected merkle store.
+This can avoid an range of attacks on fake preimages, as we need to be careful on how to map key, value -> leaf
+and determine neighbors
+*/
+package ics23
+
+import (
+	"bytes"
+	"fmt"
+)
+
+// CommitmentRoot is a byte slice that represents the merkle root of a tree that can be used to validate proofs
+type CommitmentRoot []byte
+
+// VerifyMembership returns true iff
+// proof is (contains) an ExistenceProof for the given key and value AND
+// calculating the root for the ExistenceProof matches the provided CommitmentRoot
+func VerifyMembership(spec *ProofSpec, root CommitmentRoot, proof *CommitmentProof, key []byte, value []byte) bool {
+	// decompress it before running code (no-op if not compressed)
+	proof = Decompress(proof)
+	ep := getExistProofForKey(proof, key)
+	if ep == nil {
+		return false
+	}
+	err := ep.Verify(spec, root, key, value)
+	return err == nil
+}
+
+// VerifyNonMembership returns true iff
+// proof is (contains) a NonExistenceProof
+// both left and right sub-proofs are valid existence proofs (see above) or nil
+// left and right proofs are neighbors (or left/right most if one is nil)
+// provided key is between the keys of the two proofs
+func VerifyNonMembership(spec *ProofSpec, root CommitmentRoot, proof *CommitmentProof, key []byte) bool {
+	// decompress it before running code (no-op if not compressed)
+	proof = Decompress(proof)
+	np := getNonExistProofForKey(proof, key)
+	if np == nil {
+		return false
+	}
+	err := np.Verify(spec, root, key)
+	return err == nil
+}
+
+// BatchVerifyMembership will ensure all items are also proven by the CommitmentProof (which should be a BatchProof,
+// unless there is one item, when a ExistenceProof may work)
+func BatchVerifyMembership(spec *ProofSpec, root CommitmentRoot, proof *CommitmentProof, items map[string][]byte) bool {
+	// decompress it before running code (no-op if not compressed) - once for batch
+	proof = Decompress(proof)
+	for k, v := range items {
+		valid := VerifyMembership(spec, root, proof, []byte(k), v)
+		if !valid {
+			return false
+		}
+	}
+	return true
+}
+
+// BatchVerifyNonMembership will ensure all items are also proven to not be in the Commitment by the CommitmentProof
+// (which should be a BatchProof, unless there is one item, when a NonExistenceProof may work)
+func BatchVerifyNonMembership(spec *ProofSpec, root CommitmentRoot, proof *CommitmentProof, keys [][]byte) bool {
+	// decompress it before running code (no-op if not compressed) - once for batch
+	proof = Decompress(proof)
+	for _, k := range keys {
+		valid := VerifyNonMembership(spec, root, proof, k)
+		if !valid {
+			return false
+		}
+	}
+	return true
+}
+
+// CombineProofs takes a number of commitment proofs (simple or batch) and
+// converts them into a batch and compresses them.
+//
+// This is designed for proof generation libraries to create efficient batches
+func CombineProofs(proofs []*CommitmentProof) (*CommitmentProof, error) {
+	var entries []*BatchEntry
+
+	for _, proof := range proofs {
+		if ex := proof.GetExist(); ex != nil {
+			entry := &BatchEntry{
+				Proof: &BatchEntry_Exist{
+					Exist: ex,
+				},
+			}
+			entries = append(entries, entry)
+		} else if non := proof.GetNonexist(); non != nil {
+			entry := &BatchEntry{
+				Proof: &BatchEntry_Nonexist{
+					Nonexist: non,
+				},
+			}
+			entries = append(entries, entry)
+		} else if batch := proof.GetBatch(); batch != nil {
+			entries = append(entries, batch.Entries...)
+		} else if comp := proof.GetCompressed(); comp != nil {
+			decomp := Decompress(proof)
+			entries = append(entries, decomp.GetBatch().Entries...)
+		} else {
+			return nil, fmt.Errorf("proof neither exist or nonexist: %#v", proof.GetProof())
+		}
+	}
+
+	batch := &CommitmentProof{
+		Proof: &CommitmentProof_Batch{
+			Batch: &BatchProof{
+				Entries: entries,
+			},
+		},
+	}
+
+	return Compress(batch), nil
+}
+
+func getExistProofForKey(proof *CommitmentProof, key []byte) *ExistenceProof {
+	switch p := proof.Proof.(type) {
+	case *CommitmentProof_Exist:
+		ep := p.Exist
+		if bytes.Equal(ep.Key, key) {
+			return ep
+		}
+	case *CommitmentProof_Batch:
+		for _, sub := range p.Batch.Entries {
+			if ep := sub.GetExist(); ep != nil && bytes.Equal(ep.Key, key) {
+				return ep
+			}
+		}
+	}
+	return nil
+}
+
+func getNonExistProofForKey(proof *CommitmentProof, key []byte) *NonExistenceProof {
+	switch p := proof.Proof.(type) {
+	case *CommitmentProof_Nonexist:
+		np := p.Nonexist
+		if isLeft(np.Left, key) && isRight(np.Right, key) {
+			return np
+		}
+	case *CommitmentProof_Batch:
+		for _, sub := range p.Batch.Entries {
+			if np := sub.GetNonexist(); np != nil && isLeft(np.Left, key) && isRight(np.Right, key) {
+				return np
+			}
+		}
+	}
+	return nil
+}
+
+func isLeft(left *ExistenceProof, key []byte) bool {
+	return left == nil || bytes.Compare(left.Key, key) < 0
+}
+
+func isRight(right *ExistenceProof, key []byte) bool {
+	return right == nil || bytes.Compare(right.Key, key) > 0
+}