feat: Add API TraverseStateChanges to extract state changes from ia…

…vl versions (cosmos#654)

Co-authored-by: Marko <marbar3778@yahoo.com>

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Improvements
 
+- [#654](https://github.com/cosmos/iavl/pull/654) Add API `TraverseStateChanges` to extract state changes from iavl versions.
 - [#726](https://github.com/cosmos/iavl/pull/726) Make `KVPair` and `ChangeSet` serializable with protobuf.
 
 ## 0.20.0 (March 14, 2023)

diff_test.go

@@ -21,35 +21,44 @@ func TestDiffRoundTrip(t *testing.T) {
 	db := db.NewMemDB()
 	tree, err := NewMutableTree(db, 0, true)
 	require.NoError(t, err)
-	for i := range changeSets {
-		v, err := tree.SaveChangeSet(changeSets[i])
+	for _, cs := range changeSets {
+		for _, pair := range cs.Pairs {
+			if pair.Delete {
+				_, removed, err := tree.Remove(pair.Key)
+				require.True(t, removed)
+				require.NoError(t, err)
+			} else {
+				_, err := tree.Set(pair.Key, pair.Value)
+				require.NoError(t, err)
+			}
+		}
+		_, _, err := tree.SaveVersion()
 		require.NoError(t, err)
-		require.Equal(t, int64(i+1), v)
 	}
 
 	// extract change sets from db
-	var extractChangeSets []*ChangeSet
+	var extractChangeSets []ChangeSet
 	tree2 := NewImmutableTree(db, 0, true)
-	err = tree2.ndb.traverseStateChanges(0, math.MaxInt64, func(version int64, changeSet *ChangeSet) error {
-		extractChangeSets = append(extractChangeSets, changeSet)
+	err = tree2.TraverseStateChanges(0, math.MaxInt64, func(version int64, changeSet *ChangeSet) error {
+		extractChangeSets = append(extractChangeSets, *changeSet)
 		return nil
 	})
 	require.NoError(t, err)
 	require.Equal(t, changeSets, extractChangeSets)
 }
 
-func genChangeSets(r *rand.Rand, n int) []*ChangeSet {
-	var changeSets []*ChangeSet
+func genChangeSets(r *rand.Rand, n int) []ChangeSet {
+	var changeSets []ChangeSet
 
 	for i := 0; i < n; i++ {
-		items := make(map[string]*KVPair)
+		items := make(map[string]KVPair)
 		start, count, step := r.Int63n(1000), r.Int63n(1000), r.Int63n(10)
 		for i := start; i < start+count*step; i += step {
 			value := make([]byte, 8)
 			binary.LittleEndian.PutUint64(value, uint64(i))
 
 			key := fmt.Sprintf("test-%d", i)
-			items[key] = &KVPair{
+			items[key] = KVPair{
 				Key:   []byte(key),
 				Value: value,
 			}
@@ -65,7 +74,7 @@ func genChangeSets(r *rand.Rand, n int) []*ChangeSet {
 				if pair.Delete {
 					continue
 				}
-				items[string(pair.Key)] = &KVPair{
+				items[string(pair.Key)] = KVPair{
 					Key:    pair.Key,
 					Delete: true,
 				}
@@ -77,7 +86,7 @@ func genChangeSets(r *rand.Rand, n int) []*ChangeSet {
 				i := r.Int63n(int64(len(lastChangeSet.Pairs)))
 				pair := lastChangeSet.Pairs[i]
 				if !pair.Delete {
-					items[string(pair.Key)] = &KVPair{
+					items[string(pair.Key)] = KVPair{
 						Key:   pair.Key,
 						Value: pair.Value,
 					}
@@ -93,10 +102,11 @@ func genChangeSets(r *rand.Rand, n int) []*ChangeSet {
 
 		var cs ChangeSet
 		for _, key := range keys {
-			cs.Pairs = append(cs.Pairs, items[key])
+			p := items[key]
+			cs.Pairs = append(cs.Pairs, &p)
 		}
 
-		changeSets = append(changeSets, &cs)
+		changeSets = append(changeSets, cs)
 	}
 	return changeSets
 }

immutable_tree.go

@@ -331,3 +331,9 @@ func (t *ImmutableTree) nodeSize() int {
 	})
 	return size
 }
+
+// TraverseStateChanges iterate the range of versions, compare each version to it's predecessor to extract the state changes of it.
+// endVersion is exclusive.
+func (t *ImmutableTree) TraverseStateChanges(startVersion, endVersion int64, fn func(version int64, changeSet *ChangeSet) error) error {
+	return t.ndb.traverseStateChanges(startVersion, endVersion, fn)
+}

nodedb.go

@@ -1118,49 +1118,34 @@ var (
 // traverseStateChanges iterate the range of versions, compare each version to it's predecessor to extract the state changes of it.
 // endVersion is exclusive, set to `math.MaxInt64` to cover the latest version.
 func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func(version int64, changeSet *ChangeSet) error) error {
-	firstVersion, err := ndb.getFirstVersion()
+	predecessor, err := ndb.getPreviousVersion(startVersion)
 	if err != nil {
 		return err
 	}
-	if startVersion < firstVersion {
-		startVersion = firstVersion
-	}
-	latestVersion, err := ndb.getLatestVersion()
+	prevRoot, err := ndb.getRoot(predecessor)
 	if err != nil {
 		return err
 	}
-	if endVersion > latestVersion {
-		endVersion = latestVersion
-	}
-
-	prevVersion := startVersion - 1
-	prevRoot, err := ndb.getRoot(prevVersion)
-	if err != nil && err != ErrVersionDoesNotExist {
-		return err
-	}
-
-	for version := startVersion; version <= endVersion; version++ {
-		root, err := ndb.getRoot(version)
-		if err != nil {
-			return err
-		}
+	return ndb.traverseRange(rootKeyFormat.Key(startVersion), rootKeyFormat.Key(endVersion), func(k, hash []byte) error {
+		var version int64
+		rootKeyFormat.Scan(k, &version)
 
 		var changeSet ChangeSet
 		receiveKVPair := func(pair *KVPair) error {
 			changeSet.Pairs = append(changeSet.Pairs, pair)
 			return nil
 		}
 
-		if err := ndb.extractStateChanges(prevVersion, prevRoot, root, receiveKVPair); err != nil {
+		if err := ndb.extractStateChanges(predecessor, prevRoot, hash, receiveKVPair); err != nil {
 			return err
 		}
 
 		if err := fn(version, &changeSet); err != nil {
 			return err
 		}
-		prevVersion = version
-		prevRoot = root
-	}
 
-	return nil
+		predecessor = version
+		prevRoot = hash
+		return nil
+	})
 }