feat: Add API TraverseStateChanges to extract state changes from iavl versions

CHANGELOG.md

@@ -9,6 +9,7 @@
 - [#586](https://github.com/cosmos/iavl/pull/586) Remove the `RangeProof` and refactor the ics23_proof to use the internal methods.
 - [#640](https://github.com/cosmos/iavl/pull/640) commit `NodeDB` batch in `LoadVersionForOverwriting`.
 - [#636](https://github.com/cosmos/iavl/pull/636) Speed up rollback method: `LoadVersionForOverwriting`.
+- [#]() Add API `TraverseStateChanges` to extract state changes from iavl versions.
 
 ## 0.19.4 (October 28, 2022)
 

diff.go

@@ -0,0 +1,68 @@
+package iavl
+
+import (
+	"bytes"
+	"sort"
+
+	ibytes "github.com/cosmos/iavl/internal/bytes"
+)
+
+// ChangeSet represents the state changes extracted from diffing iavl versions.
+type ChangeSet struct {
+	Pairs []KVPair
+}
+
+type KVPair struct {
+	Delete bool
+	Key    []byte
+	Value  []byte
+}
+
+func StateChanges(ndb *nodeDB, version int64, root []byte, successorRoot []byte) (*ChangeSet, error) {
+	curIter, err := NewNodeIterator(successorRoot, ndb)
+	if err != nil {
+		return nil, err
+	}
+
+	prevIter, err := NewNodeIterator(root, ndb)
+	if err != nil {
+		return nil, err
+	}
+
+	var changeSet []KVPair
+	sharedNodes := make(map[string]struct{})
+	newKeys := make(map[string]struct{})
+	for curIter.Valid() {
+		node := curIter.GetNode()
+		if node.version <= version {
+			sharedNodes[ibytes.UnsafeBytesToStr(node.hash)] = struct{}{}
+		} else if node.isLeaf() {
+			changeSet = append(changeSet, KVPair{Key: node.key, Value: node.value})
+			newKeys[ibytes.UnsafeBytesToStr(node.key)] = struct{}{}
+		}
+		curIter.Next(node.version <= version)
+	}
+	if err := curIter.Error(); err != nil {
+		return nil, err
+	}
+
+	for prevIter.Valid() {
+		node := prevIter.GetNode()
+		_, shared := sharedNodes[ibytes.UnsafeBytesToStr(node.hash)]
+		if !shared && node.isLeaf() {
+			_, updated := newKeys[ibytes.UnsafeBytesToStr(node.key)]
+			if !updated {
+				changeSet = append(changeSet, KVPair{Delete: true, Key: node.key})
+			}
+		}
+		prevIter.Next(shared)
+	}
+	if err := prevIter.Error(); err != nil {
+		return nil, err
+	}
+
+	sort.Slice(changeSet, func(i, j int) bool {
+		return bytes.Compare(changeSet[i].Key, changeSet[j].Key) == -1
+	})
+	return &ChangeSet{Pairs: changeSet}, nil
+}

immutable_tree.go

@@ -331,3 +331,8 @@ func (t *ImmutableTree) nodeSize() int {
 	})
 	return size
 }
+
+// TraverseStateChanges iterate the version range and extract state changes from diffing the adjacent versions.
+func (t *ImmutableTree) TraverseStateChanges(startVersion, endVersion int64, fn func(version int64, changeSet *ChangeSet) error) error {
+	return t.ndb.traverseStateChanges(startVersion, endVersion, fn)
+}

iterator.go

@@ -259,3 +259,74 @@ func (iter *Iterator) Error() error {
 func (iter *Iterator) IsFast() bool {
 	return false
 }
+
+// NodeIterator is an iterator for nodeDB to traverse a tree.
+type NodeIterator struct {
+	nodesToVisit []*Node
+	ndb          *nodeDB
+	err          error
+}
+
+// NewNodeIterator returns a new NodeIterator.
+func NewNodeIterator(root []byte, ndb *nodeDB) (*NodeIterator, error) {
+	if len(root) == 0 {
+		return &NodeIterator{
+			nodesToVisit: []*Node{},
+			ndb:          ndb,
+		}, nil
+	}
+
+	node, err := ndb.GetNode(root)
+	if err != nil {
+		return nil, err
+	}
+
+	return &NodeIterator{
+		nodesToVisit: []*Node{node},
+		ndb:          ndb,
+	}, nil
+}
+
+// GetNode returns the current visiting node.
+func (iter *NodeIterator) GetNode() *Node {
+	return iter.nodesToVisit[len(iter.nodesToVisit)-1]
+}
+
+// Valid checks if the validator is valid.
+func (iter *NodeIterator) Valid() bool {
+	if iter.err != nil {
+		return false
+	}
+	return len(iter.nodesToVisit) > 0
+}
+
+// Error returns an error if any errors.
+func (iter *NodeIterator) Error() error {
+	return iter.err
+}
+
+// Next moves to the next iterator.
+func (iter *NodeIterator) Next(isSkipped bool) {
+	if !iter.Valid() {
+		return
+	}
+	node := iter.GetNode()
+	iter.nodesToVisit = iter.nodesToVisit[:len(iter.nodesToVisit)-1]
+	if !isSkipped {
+		if node.isLeaf() {
+			return
+		}
+		leftNode, err := iter.ndb.GetNode(node.leftHash)
+		if err != nil {
+			iter.err = err
+			return
+		}
+		iter.nodesToVisit = append(iter.nodesToVisit, leftNode)
+		rightNode, err := iter.ndb.GetNode(node.rightHash)
+		if err != nil {
+			iter.err = err
+			return
+		}
+		iter.nodesToVisit = append(iter.nodesToVisit, rightNode)
+	}
+}

nodedb.go

@@ -1042,6 +1042,39 @@ func (ndb *nodeDB) traverseNodes(fn func(hash []byte, node *Node) error) error {
 	return nil
 }
 
+func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func(version int64, changeSet *ChangeSet) error) error {
+	if endVersion == 0 {
+		latestVersion, err := ndb.getLatestVersion()
+		if err != nil {
+			return err
+		}
+		endVersion = latestVersion + 1
+	}
+
+	predecessor, err := ndb.getPreviousVersion(startVersion)
+	if err != nil {
+		return err
+	}
+	prevRoot, err := ndb.getRoot(predecessor)
+	if err != nil {
+		return err
+	}
+	return ndb.traverseRange(rootKeyFormat.Key(startVersion), rootKeyFormat.Key(endVersion), func(k, hash []byte) error {
+		var version int64
+		rootKeyFormat.Scan(k, &version)
+		changeSet, err := StateChanges(ndb, predecessor, prevRoot, hash)
+		if err != nil {
+			return err
+		}
+		if err := fn(version, changeSet); err != nil {
+			return err
+		}
+		predecessor = version
+		prevRoot = hash
+		return nil
+	})
+}
+
 func (ndb *nodeDB) String() (string, error) {
 	buf := bufPool.Get().(*bytes.Buffer)
 	defer bufPool.Put(buf)