Merge PR #5435: Added iterator that allows to read only requested values

CHANGELOG.md

@@ -93,6 +93,7 @@ increased significantly due to modular `AnteHandler` support. Increase GasLimit
 
 ### Features
 
+* (store) [\#5435](https://github.com/cosmos/cosmos-sdk/pull/5435) New iterator for paginated requests. Iterator limits DB reads to the range of the requested page.
 * (x/evidence) [\#5240](https://github.com/cosmos/cosmos-sdk/pull/5240) Initial implementation of the `x/evidence` module.
 * (cli) [\#5212](https://github.com/cosmos/cosmos-sdk/issues/5212) The `q gov proposals` command now supports pagination.
 * (store) [\#4724](https://github.com/cosmos/cosmos-sdk/issues/4724) Multistore supports substore migrations upon load. New `rootmulti.Store.LoadLatestVersionAndUpgrade` method in
@@ -2804,3 +2805,5 @@ BUG FIXES:
 [v0.37.1]: https://github.com/cosmos/cosmos-sdk/releases/tag/v0.37.1
 [v0.37.0]: https://github.com/cosmos/cosmos-sdk/releases/tag/v0.37.0
 [v0.36.0]: https://github.com/cosmos/cosmos-sdk/releases/tag/v0.36.0
+
+

store/types/iterator.go

@@ -0,0 +1,61 @@
+package types
+
+import (
+	"fmt"
+)
+
+// KVStorePrefixIteratorPaginated returns iterator over items in the selected page.
+// Items iterated and skipped in ascending order.
+func KVStorePrefixIteratorPaginated(kvs KVStore, prefix []byte, page, limit uint) Iterator {
+	pi := &PaginatedIterator{
+		Iterator: KVStorePrefixIterator(kvs, prefix),
+		page:     page,
+		limit:    limit,
+	}
+	pi.skip()
+	return pi
+}
+
+// KVStoreReversePrefixIteratorPaginated returns iterator over items in the selected page.
+// Items iterated and skipped in descending order.
+func KVStoreReversePrefixIteratorPaginated(kvs KVStore, prefix []byte, page, limit uint) Iterator {
+	pi := &PaginatedIterator{
+		Iterator: KVStoreReversePrefixIterator(kvs, prefix),
+		page:     page,
+		limit:    limit,
+	}
+	pi.skip()
+	return pi
+}
+
+// PaginatedIterator is a wrapper around Iterator that iterates over values starting for given page and limit.
+type PaginatedIterator struct {
+	Iterator
+
+	page, limit uint // provided during initialization
+	iterated    uint // incremented in a call to Next
+
+}
+
+func (pi *PaginatedIterator) skip() {
+	for i := (pi.page - 1) * pi.limit; i > 0 && pi.Iterator.Valid(); i-- {
+		pi.Iterator.Next()
+	}
+}
+
+// Next will panic after limit is reached.
+func (pi *PaginatedIterator) Next() {
+	if !pi.Valid() {
+		panic(fmt.Sprintf("PaginatedIterator reached limit %d", pi.limit))
+	}
+	pi.Iterator.Next()
+	pi.iterated++
+}
+
+// Valid if below limit and underlying iterator is valid.
+func (pi *PaginatedIterator) Valid() bool {
+	if pi.iterated >= pi.limit {
+		return false
+	}
+	return pi.Iterator.Valid()
+}

store/types/iterator_test.go

@@ -0,0 +1,119 @@
+package types_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	dbm "github.com/tendermint/tm-db"
+
+	"github.com/cosmos/cosmos-sdk/store/iavl"
+	"github.com/cosmos/cosmos-sdk/store/types"
+)
+
+func newMemTestKVStore(t *testing.T) types.KVStore {
+	db := dbm.NewMemDB()
+	store, err := iavl.LoadStore(db, types.CommitID{}, types.PruneNothing, false)
+	require.NoError(t, err)
+	return store
+}
+
+func TestPaginatedIterator(t *testing.T) {
+	kvs := newMemTestKVStore(t)
+	total := 10
+	lth := total - 1
+	asc := make([][]byte, total)
+	desc := make([][]byte, total)
+	// store returns values in lexicographic order (or reverse lex order)
+	for i := 0; i < total; i++ {
+		key := []byte{byte(i)}
+		kvs.Set(key, key)
+		asc[i] = key
+		desc[lth-i] = key
+	}
+	type testCase struct {
+		desc        string
+		page, limit uint
+		result      [][]byte
+		reverse     bool
+	}
+	for _, tc := range []testCase{
+		{
+			desc:   "FirstChunk",
+			page:   1,
+			limit:  4,
+			result: asc[:4],
+		},
+		{
+			desc:   "SecondChunk",
+			page:   2,
+			limit:  4,
+			result: asc[4:8],
+		},
+		{
+			desc:   "ThirdChunkHalf",
+			page:   3,
+			limit:  4,
+			result: asc[8:],
+		},
+		{
+			desc:   "OverLimit",
+			page:   10,
+			limit:  10,
+			result: [][]byte{},
+		},
+		{
+			desc:   "ZeroLimit",
+			page:   1,
+			result: [][]byte{},
+		},
+		{
+			desc:    "ReverseFirstChunk",
+			page:    1,
+			limit:   6,
+			result:  desc[:6],
+			reverse: true,
+		},
+		{
+			desc:    "ReverseSecondChunk",
+			page:    2,
+			limit:   6,
+			result:  desc[6:],
+			reverse: true,
+		},
+	} {
+		tc := tc
+		t.Run(tc.desc, func(t *testing.T) {
+			var iter types.Iterator
+			if tc.reverse {
+				iter = types.KVStoreReversePrefixIteratorPaginated(kvs, nil, tc.page, tc.limit)
+			} else {
+				iter = types.KVStorePrefixIteratorPaginated(kvs, nil, tc.page, tc.limit)
+			}
+			defer iter.Close()
+
+			result := [][]byte{}
+			for ; iter.Valid(); iter.Next() {
+				result = append(result, iter.Key())
+			}
+
+			require.Equal(t, tc.result, result)
+			require.False(t, iter.Valid())
+		})
+	}
+}
+
+func TestPaginatedIteratorPanicIfInvalid(t *testing.T) {
+	kvs := newMemTestKVStore(t)
+
+	iter := types.KVStorePrefixIteratorPaginated(kvs, nil, 1, 1)
+	defer iter.Close()
+	require.False(t, iter.Valid())
+	require.Panics(t, func() { iter.Next() }) // "iterator is empty"
+
+	kvs.Set([]byte{1}, []byte{})
+
+	iter = types.KVStorePrefixIteratorPaginated(kvs, nil, 1, 0)
+	defer iter.Close()
+	require.False(t, iter.Valid())
+	require.Panics(t, func() { iter.Next() }) // "not empty but limit is zero"
+}

types/store.go

@@ -40,6 +40,18 @@ func KVStoreReversePrefixIterator(kvs KVStore, prefix []byte) Iterator {
 	return types.KVStoreReversePrefixIterator(kvs, prefix)
 }
 
+// KVStorePrefixIteratorPaginated returns iterator over items in the selected page.
+// Items iterated and skipped in ascending order.
+func KVStorePrefixIteratorPaginated(kvs KVStore, prefix []byte, page, limit uint) Iterator {
+	return types.KVStorePrefixIteratorPaginated(kvs, prefix, page, limit)
+}
+
+// KVStoreReversePrefixIteratorPaginated returns iterator over items in the selected page.
+// Items iterated and skipped in descending order.
+func KVStoreReversePrefixIteratorPaginated(kvs KVStore, prefix []byte, page, limit uint) Iterator {
+	return types.KVStorePrefixIteratorPaginated(kvs, prefix, page, limit)
+}
+
 // DiffKVStores compares two KVstores and returns all the key/value pairs
 // that differ from one another. It also skips value comparison for a set of provided prefixes
 func DiffKVStores(a KVStore, b KVStore, prefixesToSkip [][]byte) (kvAs, kvBs []cmn.KVPair) {