feat: Add ics23 proof tools: ics23-{iavl,tendermint,smt}

store/tools/ics23/Makefile

@@ -0,0 +1,44 @@
+GO_RUN= go run -mod=readonly
+GENDIR ?= ./testdata
+
+.PHONY: testgen testgen-iavl testgen-smt testgen-simple
+
+# make sure we turn on go modules
+export GO111MODULE := on
+
+# Usage: GENDIR=/path/to/ics23/testdata/iavl make testgen-iavl
+
+testgen: testgen-iavl testgen-smt testgen-simple
+
+testgen-iavl:
+	@mkdir -p "$(GENDIR)"
+	$(GO_RUN) ./iavl/cmd/testgen-iavl exist left 987 > "$(GENDIR)"/exist_left.json
+	$(GO_RUN) ./iavl/cmd/testgen-iavl exist middle 812 > "$(GENDIR)"/exist_middle.json
+	$(GO_RUN) ./iavl/cmd/testgen-iavl exist right 1261 > "$(GENDIR)"/exist_right.json
+	$(GO_RUN) ./iavl/cmd/testgen-iavl nonexist left 813 > "$(GENDIR)"/nonexist_left.json
+	$(GO_RUN) ./iavl/cmd/testgen-iavl nonexist middle 691 > "$(GENDIR)"/nonexist_middle.json
+	$(GO_RUN) ./iavl/cmd/testgen-iavl nonexist right 1535 > "$(GENDIR)"/nonexist_right.json
+	$(GO_RUN) ./iavl/cmd/testgen-iavl batch 1801 20 0 > "$(GENDIR)"/batch_exist.json
+	$(GO_RUN) ./iavl/cmd/testgen-iavl batch 1807 0 20 > "$(GENDIR)"/batch_nonexist.json
+
+testgen-smt:
+	@mkdir -p "$(GENDIR)"
+	$(GO_RUN) ./smt/cmd/testgen-smt exist left 987 > "$(GENDIR)"/exist_left.json
+	$(GO_RUN) ./smt/cmd/testgen-smt exist middle 812 > "$(GENDIR)"/exist_middle.json
+	$(GO_RUN) ./smt/cmd/testgen-smt exist right 1261 > "$(GENDIR)"/exist_right.json
+	$(GO_RUN) ./smt/cmd/testgen-smt nonexist left 813 > "$(GENDIR)"/nonexist_left.json
+	$(GO_RUN) ./smt/cmd/testgen-smt nonexist middle 691 > "$(GENDIR)"/nonexist_middle.json
+	$(GO_RUN) ./smt/cmd/testgen-smt nonexist right 1535 > "$(GENDIR)"/nonexist_right.json
+	$(GO_RUN) ./smt/cmd/testgen-smt batch 1801 20 0 > "$(GENDIR)"/batch_exist.json
+	$(GO_RUN) ./smt/cmd/testgen-smt batch 1807 0 20 > "$(GENDIR)"/batch_nonexist.json
+
+testgen-simple:
+	@mkdir -p "$(GENDIR)"
+	$(GO_RUN) ./tendermint/cmd/testgen-simple exist left 987 > "$(GENDIR)"/exist_left.json
+	$(GO_RUN) ./tendermint/cmd/testgen-simple exist middle 812 > "$(GENDIR)"/exist_middle.json
+	$(GO_RUN) ./tendermint/cmd/testgen-simple exist right 1261 > "$(GENDIR)"/exist_right.json
+	$(GO_RUN) ./tendermint/cmd/testgen-simple nonexist left 813 > "$(GENDIR)"/nonexist_left.json
+	$(GO_RUN) ./tendermint/cmd/testgen-simple nonexist middle 691 > "$(GENDIR)"/nonexist_middle.json
+	$(GO_RUN) ./tendermint/cmd/testgen-simple nonexist right 1535 > "$(GENDIR)"/nonexist_right.json
+	$(GO_RUN) ./tendermint/cmd/testgen-simple batch 1801 20 0 > "$(GENDIR)"/batch_exist.json
+	$(GO_RUN) ./tendermint/cmd/testgen-simple batch 1807 0 20 > "$(GENDIR)"/batch_nonexist.json

store/tools/ics23/go.mod

@@ -0,0 +1,31 @@
+module github.com/cosmos/cosmos-sdk/store/tools/ics23
+
+go 1.17
+
+require (
+	github.com/confio/ics23/go v0.7.0
+	github.com/lazyledger/smt v0.2.1-0.20210709230900-03ea40719554
+	github.com/tendermint/iavl v0.13.2
+	github.com/tendermint/tendermint v0.33.2
+	github.com/tendermint/tm-db v0.5.0
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/etcd-io/bbolt v1.3.3 // indirect
+	github.com/gogo/protobuf v1.3.1 // indirect
+	github.com/golang/protobuf v1.3.4 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/google/btree v1.0.0 // indirect
+	github.com/jmhodges/levigo v1.0.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d // indirect
+	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
+	github.com/tendermint/go-amino v0.14.1 // indirect
+	golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect
+	golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 // indirect
+)
+
+replace github.com/cosmos/cosmos-sdk/store/tools/ics23 => ./
+
+replace github.com/confio/ics23/go => /home/roy/vulcanize/repo/ics23/go

store/tools/ics23/iavl/Makefile

@@ -0,0 +1,18 @@
+.PHONY: testgen
+
+GENDIR ?= ./testdata
+
+# make sure we turn on go modules
+export GO111MODULE := on
+
+# Usage: GENDIR=../ics23/testdata/iavl make testgen
+testgen:
+	@mkdir -p "$(GENDIR)"
+	go run -mod=readonly ./cmd/testgen-iavl exist left 987 > "$(GENDIR)"/exist_left.json
+	go run -mod=readonly ./cmd/testgen-iavl exist middle 812 > "$(GENDIR)"/exist_middle.json
+	go run -mod=readonly ./cmd/testgen-iavl exist right 1261 > "$(GENDIR)"/exist_right.json
+	go run -mod=readonly ./cmd/testgen-iavl nonexist left 813 > "$(GENDIR)"/nonexist_left.json
+	go run -mod=readonly ./cmd/testgen-iavl nonexist middle 691 > "$(GENDIR)"/nonexist_middle.json
+	go run -mod=readonly ./cmd/testgen-iavl nonexist right 1535 > "$(GENDIR)"/nonexist_right.json
+	go run -mod=readonly ./cmd/testgen-iavl batch 1801 20 0 > "$(GENDIR)"/batch_exist.json
+	go run -mod=readonly ./cmd/testgen-iavl batch 1807 0 20 > "$(GENDIR)"/batch_nonexist.json

store/tools/ics23/iavl/README.md

@@ -0,0 +1,44 @@
+# Proofs IAVL
+
+This is a demo project to show converting proofs from tendermint/iavl into the format
+specified in confio/proofs and validating that they still work.
+
+## Library usage
+
+It exposes a two main functions :
+
+`func CreateMembershipProof(tree *iavl.MutableTree, key []byte) (*proofs.CommitmentProof, error)`
+produces a CommitmentProof that the given key exists in the iavl tree (and contains the
+current value). This returns an error if the key does not exist in the tree.
+
+`func CreateNonMembershipProof(tree *iavl.MutableTree, key []byte) (*proofs.CommitmentProof, error)`
+produces a CommitmentProof that the given key doesn't exist in the iavl tree. 
+This returns an error if the key does not exist in the tree.
+
+Generalized range proofs are lower in priority, as they are just an optimization of the
+two basic proof types, and don't provide any additional capabilities.
+We will soon add some `Batch` capabilities that can represent these.
+
+## CLI usage
+
+We also expose a simple script to generate test data for the confio proofs package.
+
+```shell
+go install ./cmd/testgen-iavl
+testgen-iavl
+```
+
+Will output some json data, from a randomly generated merkle tree each time.
+
+```json
+{
+  "existence": "0a146f65436a684273735a34567543774b567a435963121e76616c75655f666f725f6f65436a684273735a34567543774b567a4359631a0d0a0b0801180120012a030002021a2d122b08011204020402201a2120d307032505383dee34ea9eadf7649c31d1ce294b6d62b273d804da478ac161da1a2d122b08011204040802201a2120306b7d51213bd93bac17c5ee3d727ec666300370b19fd55cc13d7341dc589a991a2b12290801122508160220857103d59863ac55d1f34008a681f837c01975a223c0f54883a05a446d49c7c6201a2b1229080112250a2202204498eb5c93e40934bc8bad9626f19e333c1c0be4541b9098f139585c3471bae2201a2d122b080112040e6c02201a212022648db12dbf830485cc41435ecfe37bcac26c6c305ac4304f649977ddc339d51a2c122a0801122610c60102204e0b7996a7104f5b1ac1a2caa0704c4b63f60112e0e13763b2ba03f40a54e845201a2c122a08011226129003022017858e28e0563f7252eaca19acfc1c3828c892e635f76f971b3fbdc9bbd2742e20",
+  "root": "cea07656c77e8655521f4c904730cf4649242b8e482be786b2b220a15150d5f0"
+}
+```
+
+`"root"` is the hex-encoded root hash of the merkle tree
+
+`"existence"` is the hex-encoding of the protobuf binary encoding of a `proofs.ExistenceProof` object. This contains a (key, value) pair,
+along with all steps to reach the root hash. This provides a non-trivial test case, to ensure client in multiple languages can verify the
+protobuf proofs we generate from the iavl tree  

store/tools/ics23/iavl/cmd/testgen-iavl/main.go

@@ -0,0 +1,242 @@
+package main
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strconv"
+
+	ics23 "github.com/confio/ics23/go"
+
+	iavlproofs "github.com/cosmos/cosmos-sdk/store/tools/ics23/iavl"
+	"github.com/cosmos/cosmos-sdk/store/tools/ics23/iavl/helpers"
+)
+
+/**
+testgen-iavl will generate a json struct on stdout (meant to be saved to file for testdata).
+this will be an auto-generated existence proof in the form:
+
+{
+	"root": "<hex encoded root hash of tree>",
+	"key": "<hex encoded key to prove>",
+	"value": "<hex encoded value to prove> (empty on non-existence)",
+	"proof": "<hex encoded protobuf marshaling of a CommitmentProof>"
+}
+
+It accepts two or three arguments (optional size: default 400)
+
+  testgen-iavl [exist|nonexist] [left|right|middle] <size>
+
+If you make a batch, we have a different format:
+
+{
+	"root": "<hex encoded root hash of tree>",
+	"proof": "<hex encoded protobuf marshaling of a CommitmentProof (Compressed Batch)>",
+	"items": [{
+		"key": "<hex encoded key to prove>",
+		"value": "<hex encoded value to prove> (empty on non-existence)",
+	}, ...]
+}
+
+The batch variant accepts 5 arguments:
+
+  testgen-iavl [batch] [size] [num exist] [num nonexist]
+**/
+
+func main() {
+	if os.Args[1] == "batch" {
+		err := doBatch(os.Args[2:])
+		if err != nil {
+			fmt.Printf("%+v\n", err)
+			fmt.Println("Usage: testgen-iavl [batch] [size] [# exist] [# nonexist]")
+			os.Exit(1)
+		}
+		return
+	}
+
+	exist, loc, size, err := parseArgs(os.Args)
+	if err != nil {
+		fmt.Printf("%+v\n", err)
+		fmt.Println("Usage: testgen-iavl [exist|nonexist] [left|right|middle] <size>")
+		os.Exit(1)
+	}
+
+	tree, allkeys, err := helpers.BuildTree(size)
+	if err != nil {
+		fmt.Printf("%+v\n", err)
+		fmt.Println("Usage: testgen-iavl [exist|nonexist] [left|right|middle] <size>")
+		os.Exit(1)
+	}
+	root := tree.WorkingHash()
+
+	var key, value []byte
+	if exist {
+		key = helpers.GetKey(allkeys, loc)
+		_, value = tree.Get(key)
+	} else {
+		key = helpers.GetNonKey(allkeys, loc)
+	}
+
+	var proof *ics23.CommitmentProof
+	if exist {
+		proof, err = iavlproofs.CreateMembershipProof(tree, key)
+	} else {
+		proof, err = iavlproofs.CreateNonMembershipProof(tree, key)
+	}
+	if err != nil {
+		fmt.Printf("Error: create proof: %+v\n", err)
+		os.Exit(1)
+	}
+
+	binary, err := proof.Marshal()
+	if err != nil {
+		fmt.Printf("Error: protobuf marshal: %+v\n", err)
+		os.Exit(1)
+	}
+
+	res := map[string]interface{}{
+		"root":  hex.EncodeToString(root),
+		"key":   hex.EncodeToString(key),
+		"value": hex.EncodeToString(value),
+		"proof": hex.EncodeToString(binary),
+	}
+	out, err := json.MarshalIndent(res, "", "  ")
+	if err != nil {
+		fmt.Printf("Error: json encoding: %+v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Println(string(out))
+}
+
+func parseArgs(args []string) (exist bool, loc helpers.Where, size int, err error) {
+	if len(args) != 3 && len(args) != 4 {
+		err = fmt.Errorf("Insufficient args")
+		return
+	}
+
+	switch args[1] {
+	case "exist":
+		exist = true
+	case "nonexist":
+		exist = false
+	default:
+		err = fmt.Errorf("Invalid arg: %s", args[1])
+		return
+	}
+
+	switch args[2] {
+	case "left":
+		loc = helpers.Left
+	case "middle":
+		loc = helpers.Middle
+	case "right":
+		loc = helpers.Right
+	default:
+		err = fmt.Errorf("Invalid arg: %s", args[2])
+		return
+	}
+
+	size = 400
+	if len(args) == 4 {
+		size, err = strconv.Atoi(args[3])
+	}
+
+	return
+}
+
+type item struct {
+	Key   string `json:"key"`
+	Value string `json:"value"`
+}
+
+func doBatch(args []string) error {
+	size, exist, nonexist, err := parseBatchArgs(args)
+	if err != nil {
+		return err
+	}
+
+	tree, allkeys, err := helpers.BuildTree(size)
+	if err != nil {
+		return err
+	}
+	root := tree.WorkingHash()
+
+	items := []item{}
+	proofs := []*ics23.CommitmentProof{}
+
+	for i := 0; i < exist; i++ {
+		key := []byte(helpers.GetKey(allkeys, helpers.Middle))
+		_, value := tree.Get(key)
+		proof, err := iavlproofs.CreateMembershipProof(tree, key)
+		if err != nil {
+			return fmt.Errorf("create proof: %+v", err)
+		}
+		proofs = append(proofs, proof)
+		item := item{
+			Key:   hex.EncodeToString(key),
+			Value: hex.EncodeToString(value),
+		}
+		items = append(items, item)
+	}
+
+	for i := 0; i < nonexist; i++ {
+		key := []byte(helpers.GetNonKey(allkeys, helpers.Middle))
+		proof, err := iavlproofs.CreateNonMembershipProof(tree, key)
+		if err != nil {
+			return fmt.Errorf("create proof: %+v", err)
+		}
+		proofs = append(proofs, proof)
+		item := item{
+			Key: hex.EncodeToString(key),
+		}
+		items = append(items, item)
+	}
+
+	// combine all proofs into batch and compress
+	proof, err := ics23.CombineProofs(proofs)
+	if err != nil {
+		fmt.Printf("Error: combine proofs: %+v\n", err)
+		os.Exit(1)
+	}
+
+	binary, err := proof.Marshal()
+	if err != nil {
+		fmt.Printf("Error: protobuf marshal: %+v\n", err)
+		os.Exit(1)
+	}
+
+	res := map[string]interface{}{
+		"root":  hex.EncodeToString(root),
+		"items": items,
+		"proof": hex.EncodeToString(binary),
+	}
+	out, err := json.MarshalIndent(res, "", "  ")
+	if err != nil {
+		fmt.Printf("Error: json encoding: %+v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Println(string(out))
+
+	return nil
+}
+
+func parseBatchArgs(args []string) (size int, exist int, nonexist int, err error) {
+	if len(args) != 3 {
+		err = fmt.Errorf("Insufficient args")
+		return
+	}
+
+	size, err = strconv.Atoi(args[0])
+	if err != nil {
+		return
+	}
+	exist, err = strconv.Atoi(args[1])
+	if err != nil {
+		return
+	}
+	nonexist, err = strconv.Atoi(args[2])
+	return
+}