Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pruning Refactor #6475

Merged
merged 18 commits into from
Jun 22, 2020
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Updates
  • Loading branch information
alexanderbez committed Jun 21, 2020
commit 31b14a710744c9b22c29f7f6edf20d1fd9a79a28
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ require (
github.com/confio/ics23-tendermint v0.6.1
github.com/confio/ics23/go v0.0.0-20200604202538-6e2c36a74465
github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d
github.com/cosmos/iavl v0.13.4-0.20200620212457-07b32d6bc8b4
github.com/cosmos/ledger-cosmos-go v0.11.1
github.com/gibson042/canonicaljson-go v1.0.3
github.com/gogo/protobuf v1.3.1
Expand All @@ -38,6 +37,7 @@ require (
github.com/tendermint/btcd v0.1.1
github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15
github.com/tendermint/go-amino v0.15.1
github.com/tendermint/iavl v0.13.4-0.20200621145059-83c3470ad61d
github.com/tendermint/tendermint v0.33.5
github.com/tendermint/tm-db v0.5.1
google.golang.org/grpc v1.29.1
Expand Down
6 changes: 2 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,6 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d h1:49RLWk1j44Xu4fjHb6JFYmeUnDORVwHNkDxaQ0ctCVU=
github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y=
github.com/cosmos/iavl v0.13.4-0.20200620212457-07b32d6bc8b4 h1:EU9VPVFu+0f8TxlC0yhv1oqa7UPKGajBIO3dLDhlQDo=
github.com/cosmos/iavl v0.13.4-0.20200620212457-07b32d6bc8b4/go.mod h1:tdy3FfnhV1LQWzfxzNtgca0K+oqQAu8nqgZtiAhJdCM=
github.com/cosmos/ledger-cosmos-go v0.11.1 h1:9JIYsGnXP613pb2vPjFeMMjBI5lEDsEaF6oYorTy6J4=
github.com/cosmos/ledger-cosmos-go v0.11.1/go.mod h1:J8//BsAGTo3OC/vDLjMRFLW6q0WAaXvHnVc7ZmE8iUY=
github.com/cosmos/ledger-go v0.9.2 h1:Nnao/dLwaVTk1Q5U9THldpUMMXU94BOTWPddSmVB6pI=
Expand Down Expand Up @@ -512,8 +510,8 @@ github.com/tendermint/go-amino v0.14.1/go.mod h1:i/UKE5Uocn+argJJBb12qTZsCDBcAYM
github.com/tendermint/go-amino v0.15.1 h1:D2uk35eT4iTsvJd9jWIetzthE5C0/k2QmMFkCN+4JgQ=
github.com/tendermint/go-amino v0.15.1/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME=
github.com/tendermint/iavl v0.13.2/go.mod h1:vE1u0XAGXYjHykd4BLp8p/yivrw2PF1TuoljBcsQoGA=
github.com/tendermint/iavl v0.13.3 h1:expgBDY1MX+6/3sqrIxGChbTNf9N9aTJ67SH4bPchCs=
github.com/tendermint/iavl v0.13.3/go.mod h1:2lE7GiWdSvc7kvT78ncIKmkOjCnp6JEnSb2O7B9htLw=
github.com/tendermint/iavl v0.13.4-0.20200621145059-83c3470ad61d h1:CeGkAbISdVP2LtmxWUv69KAzvrFjndehTtKTCd/kn9E=
github.com/tendermint/iavl v0.13.4-0.20200621145059-83c3470ad61d/go.mod h1:ZftLrZ/r+rapraBmqypoFAiw8YbvTglawkmZu2grdkg=
github.com/tendermint/tendermint v0.33.2 h1:NzvRMTuXJxqSsFed2J7uHmMU5N1CVzSpfi3nCc882KY=
github.com/tendermint/tendermint v0.33.2/go.mod h1:25DqB7YvV1tN3tHsjWoc2vFtlwICfrub9XO6UBO+4xk=
github.com/tendermint/tendermint v0.33.5 h1:jYgRd9ImkzA9iOyhpmgreYsqSB6tpDa6/rXYPb8HKE8=
Expand Down
8 changes: 4 additions & 4 deletions store/cache/cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ import (
iavlstore "github.com/cosmos/cosmos-sdk/store/iavl"
"github.com/cosmos/cosmos-sdk/store/types"

"github.com/cosmos/iavl"
"github.com/stretchr/testify/require"
"github.com/tendermint/iavl"
dbm "github.com/tendermint/tm-db"
)

Expand All @@ -20,7 +20,7 @@ func TestGetOrSetStoreCache(t *testing.T) {
sKey := types.NewKVStoreKey("test")
tree, err := iavl.NewMutableTree(db, 100)
require.NoError(t, err)
store := iavlstore.UnsafeNewStore(tree, types.PruneNothing)
store := iavlstore.UnsafeNewStore(tree)
store2 := mngr.GetStoreCache(sKey, store)

require.NotNil(t, store2)
Expand All @@ -34,7 +34,7 @@ func TestUnwrap(t *testing.T) {
sKey := types.NewKVStoreKey("test")
tree, err := iavl.NewMutableTree(db, 100)
require.NoError(t, err)
store := iavlstore.UnsafeNewStore(tree, types.PruneNothing)
store := iavlstore.UnsafeNewStore(tree)
_ = mngr.GetStoreCache(sKey, store)

require.Equal(t, store, mngr.Unwrap(sKey))
Expand All @@ -48,7 +48,7 @@ func TestStoreCache(t *testing.T) {
sKey := types.NewKVStoreKey("test")
tree, err := iavl.NewMutableTree(db, 100)
require.NoError(t, err)
store := iavlstore.UnsafeNewStore(tree, types.PruneNothing)
store := iavlstore.UnsafeNewStore(tree)
kvStore := mngr.GetStoreCache(sKey, store)

for i := uint(0); i < cache.DefaultCommitKVStoreCacheSize*2; i++ {
Expand Down
40 changes: 9 additions & 31 deletions store/iavl/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (

ics23iavl "github.com/confio/ics23-iavl"
ics23 "github.com/confio/ics23/go"
"github.com/cosmos/iavl"
"github.com/tendermint/iavl"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/crypto/merkle"
tmkv "github.com/tendermint/tendermint/libs/kv"
Expand All @@ -34,38 +34,18 @@ var (

// Store Implements types.KVStore and CommitKVStore.
type Store struct {
tree Tree
pruning types.PruningOptions
tree Tree
}

// LoadStore returns an IAVL Store as a CommitKVStore. Internally, it will load the
// store's version (id) from the provided DB. An error is returned if the version
// fails to load.
func LoadStore(db dbm.DB, id types.CommitID, pruning types.PruningOptions, lazyLoading bool) (types.CommitKVStore, error) {
if !pruning.IsValid() {
return nil, fmt.Errorf("pruning options are invalid: %v", pruning)
}

var keepRecent int64

// Determine the value of keepRecent based on the following:
//
// If KeepEvery = 1, keepRecent should be 0 since there is no need to keep
// latest version in a in-memory cache.
//
// If KeepEvery > 1, keepRecent should be 1 so that state changes in between
// flushed states can be saved in the in-memory latest tree.
if pruning.KeepEvery == 1 {
keepRecent = 0
} else {
keepRecent = 1
}

func LoadStore(db dbm.DB, id types.CommitID, lazyLoading bool) (types.CommitKVStore, error) {
tree, err := iavl.NewMutableTreeWithOpts(
db,
dbm.NewMemDB(),
defaultIAVLCacheSize,
iavl.PruningOptions(pruning.KeepEvery, keepRecent),
iavl.PruningOptions(1, 0),
)
if err != nil {
return nil, err
Expand All @@ -82,8 +62,7 @@ func LoadStore(db dbm.DB, id types.CommitID, pruning types.PruningOptions, lazyL
}

return &Store{
tree: tree,
pruning: pruning,
tree: tree,
}, nil
}

Expand All @@ -93,10 +72,9 @@ func LoadStore(db dbm.DB, id types.CommitID, pruning types.PruningOptions, lazyL
// CONTRACT: The IAVL tree should be fully loaded.
// CONTRACT: PruningOptions passed in as argument must be the same as pruning options
// passed into iavl.MutableTree
func UnsafeNewStore(tree *iavl.MutableTree, po types.PruningOptions) *Store {
func UnsafeNewStore(tree *iavl.MutableTree) *Store {
return &Store{
tree: tree,
pruning: po,
tree: tree,
}
}

Expand All @@ -116,8 +94,7 @@ func (st *Store) GetImmutable(version int64) (*Store, error) {
}

return &Store{
tree: &immutableTree{iTree},
pruning: st.pruning,
tree: &immutableTree{iTree},
}, nil
}

Expand Down Expand Up @@ -348,6 +325,7 @@ func getProofFromTree(tree *iavl.MutableTree, key []byte, exists bool) *merkle.P
panic(fmt.Sprintf("unexpected error for nonexistence proof: %s", err.Error()))
}
}

op := types.NewIavlCommitmentOp(key, commitmentProof)
return &merkle.Proof{Ops: []merkle.ProofOp{op.ProofOp()}}
}
Expand Down
26 changes: 13 additions & 13 deletions store/iavl/store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import (
"fmt"
"testing"

"github.com/cosmos/iavl"
"github.com/stretchr/testify/require"
"github.com/tendermint/iavl"
abci "github.com/tendermint/tendermint/abci/types"
dbm "github.com/tendermint/tm-db"

Expand Down Expand Up @@ -52,7 +52,7 @@ func newAlohaTree(t *testing.T, db dbm.DB) (*iavl.MutableTree, types.CommitID) {
func TestGetImmutable(t *testing.T) {
db := dbm.NewMemDB()
tree, cID := newAlohaTree(t, db)
store := UnsafeNewStore(tree, types.PruneNothing)
store := UnsafeNewStore(tree)

require.True(t, tree.Set([]byte("hello"), []byte("adios")))
hash, ver, err := tree.SaveVersion()
Expand Down Expand Up @@ -82,7 +82,7 @@ func TestGetImmutable(t *testing.T) {
func TestTestGetImmutableIterator(t *testing.T) {
db := dbm.NewMemDB()
tree, cID := newAlohaTree(t, db)
store := UnsafeNewStore(tree, types.PruneNothing)
store := UnsafeNewStore(tree)

newStore, err := store.GetImmutable(cID.Version)
require.NoError(t, err)
Expand All @@ -105,7 +105,7 @@ func TestTestGetImmutableIterator(t *testing.T) {
func TestIAVLStoreGetSetHasDelete(t *testing.T) {
db := dbm.NewMemDB()
tree, _ := newAlohaTree(t, db)
iavlStore := UnsafeNewStore(tree, types.PruneNothing)
iavlStore := UnsafeNewStore(tree)

key := "hello"

Expand All @@ -130,14 +130,14 @@ func TestIAVLStoreGetSetHasDelete(t *testing.T) {
func TestIAVLStoreNoNilSet(t *testing.T) {
db := dbm.NewMemDB()
tree, _ := newAlohaTree(t, db)
iavlStore := UnsafeNewStore(tree, types.PruneNothing)
iavlStore := UnsafeNewStore(tree)
require.Panics(t, func() { iavlStore.Set([]byte("key"), nil) }, "setting a nil value should panic")
}

func TestIAVLIterator(t *testing.T) {
db := dbm.NewMemDB()
tree, _ := newAlohaTree(t, db)
iavlStore := UnsafeNewStore(tree, types.PruneNothing)
iavlStore := UnsafeNewStore(tree)
iter := iavlStore.Iterator([]byte("aloha"), []byte("hellz"))
expected := []string{"aloha", "hello"}
var i int
Expand Down Expand Up @@ -213,7 +213,7 @@ func TestIAVLReverseIterator(t *testing.T) {
tree, err := iavl.NewMutableTree(db, cacheSize)
require.NoError(t, err)

iavlStore := UnsafeNewStore(tree, types.PruneNothing)
iavlStore := UnsafeNewStore(tree)

iavlStore.Set([]byte{0x00}, []byte("0"))
iavlStore.Set([]byte{0x00, 0x00}, []byte("0 0"))
Expand Down Expand Up @@ -246,7 +246,7 @@ func TestIAVLPrefixIterator(t *testing.T) {
tree, err := iavl.NewMutableTree(db, cacheSize)
require.NoError(t, err)

iavlStore := UnsafeNewStore(tree, types.PruneNothing)
iavlStore := UnsafeNewStore(tree)

iavlStore.Set([]byte("test1"), []byte("test1"))
iavlStore.Set([]byte("test2"), []byte("test2"))
Expand Down Expand Up @@ -310,7 +310,7 @@ func TestIAVLReversePrefixIterator(t *testing.T) {
tree, err := iavl.NewMutableTree(db, cacheSize)
require.NoError(t, err)

iavlStore := UnsafeNewStore(tree, types.PruneNothing)
iavlStore := UnsafeNewStore(tree)

iavlStore.Set([]byte("test1"), []byte("test1"))
iavlStore.Set([]byte("test2"), []byte("test2"))
Expand Down Expand Up @@ -461,7 +461,7 @@ func TestIAVLNoPrune(t *testing.T) {
tree, err := iavl.NewMutableTree(db, cacheSize)
require.NoError(t, err)

iavlStore := UnsafeNewStore(tree, types.PruneNothing)
iavlStore := UnsafeNewStore(tree)
nextVersion(iavlStore)

for i := 1; i < 100; i++ {
Expand All @@ -482,7 +482,7 @@ func TestIAVLPruneEverything(t *testing.T) {
tree, err := iavl.NewMutableTreeWithOpts(db, dbm.NewMemDB(), cacheSize, iavlOpts)
require.NoError(t, err)

iavlStore := UnsafeNewStore(tree, types.PruneEverything)
iavlStore := UnsafeNewStore(tree)
nextVersion(iavlStore)

for i := 1; i < 100; i++ {
Expand All @@ -505,7 +505,7 @@ func TestIAVLStoreQuery(t *testing.T) {
tree, err := iavl.NewMutableTree(db, cacheSize)
require.NoError(t, err)

iavlStore := UnsafeNewStore(tree, types.PruneNothing)
iavlStore := UnsafeNewStore(tree)

k1, v1 := []byte("key1"), []byte("val1")
k2, v2 := []byte("key2"), []byte("val2")
Expand Down Expand Up @@ -604,7 +604,7 @@ func BenchmarkIAVLIteratorNext(b *testing.B) {
tree.Set(key, value)
}

iavlStore := UnsafeNewStore(tree, types.PruneNothing)
iavlStore := UnsafeNewStore(tree)
iterators := make([]types.Iterator, b.N/treeSize)

for i := 0; i < len(iterators); i++ {
Expand Down
2 changes: 1 addition & 1 deletion store/iavl/tree.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package iavl
import (
"fmt"

"github.com/cosmos/iavl"
"github.com/tendermint/iavl"
)

var (
Expand Down
2 changes: 1 addition & 1 deletion store/iavl/tree_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ package iavl
import (
"testing"

"github.com/cosmos/iavl"
"github.com/stretchr/testify/require"
"github.com/tendermint/iavl"
dbm "github.com/tendermint/tm-db"
)

Expand Down
4 changes: 2 additions & 2 deletions store/prefix/store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (

"github.com/stretchr/testify/require"

tiavl "github.com/cosmos/iavl"
tiavl "github.com/tendermint/iavl"
dbm "github.com/tendermint/tm-db"
)

Expand Down Expand Up @@ -90,7 +90,7 @@ func TestIAVLStorePrefix(t *testing.T) {
db := dbm.NewMemDB()
tree, err := tiavl.NewMutableTree(db, cacheSize)
require.NoError(t, err)
iavlStore := iavl.UnsafeNewStore(tree, types.PruneNothing)
iavlStore := iavl.UnsafeNewStore(tree)

testPrefixStore(t, iavlStore, []byte("test"))
}
Expand Down
2 changes: 1 addition & 1 deletion store/rootmulti/proof_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
func TestVerifyIAVLStoreQueryProof(t *testing.T) {
// Create main tree for testing.
db := dbm.NewMemDB()
iStore, err := iavl.LoadStore(db, types.CommitID{}, types.PruneNothing, false)
iStore, err := iavl.LoadStore(db, types.CommitID{}, false)
store := iStore.(*iavl.Store)
require.Nil(t, err)
store.Set([]byte("MYKEY"), []byte("MYVALUE"))
Expand Down
2 changes: 1 addition & 1 deletion store/types/iterator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (

func newMemTestKVStore(t *testing.T) types.KVStore {
db := dbm.NewMemDB()
store, err := iavl.LoadStore(db, types.CommitID{}, types.PruneNothing, false)
store, err := iavl.LoadStore(db, types.CommitID{}, false)
require.NoError(t, err)
return store
}
Expand Down