Skip to content

Commit edb321e

Browse files
yihuangmmsqe
yihuang
and
mmsqe
authored
Problem: memiavl don't support rewrite snapshot at background (#959)
* Problem: memiavl don't support rewrite snapshot at background Closes: #948 Solution: - manage multiple trees(multi-store) together. - support copy-on-write on `MemNode`s - support snapshot rewrite at background, catch-up and switch in `Commit` event. - integrate WAL Signed-off-by: yihuang <huang@crypto.com> Co-authored-by: mmsqe <tqd0800210105@gmail.com> Signed-off-by: yihuang <huang@crypto.com> --------- Signed-off-by: yihuang <huang@crypto.com> Co-authored-by: mmsqe <tqd0800210105@gmail.com>
1 parent 72c7b74 commit edb321e

33 files changed

+1602
-714
lines changed

‎go.mod‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,13 @@ require (
77
cosmossdk.io/math v1.0.0-beta.4
88
github.com/armon/go-metrics v0.4.1
99
github.com/cosmos/cosmos-sdk v0.46.11
10-
github.com/cosmos/gogoproto v1.4.2
10+
github.com/cosmos/gogoproto v1.4.7
1111
github.com/cosmos/ibc-go/v5 v5.2.0
1212
github.com/crypto-org-chain/cronos/versiondb v0.0.0
1313
github.com/ethereum/go-ethereum v1.10.26
1414
github.com/evmos/ethermint v0.6.1-0.20221101220534-a8ea4eceb6d9
1515
github.com/gogo/protobuf v1.3.3
16-
github.com/golang/protobuf v1.5.2
16+
github.com/golang/protobuf v1.5.3
1717
github.com/gorilla/mux v1.8.0
1818
github.com/grpc-ecosystem/grpc-gateway v1.16.0
1919
github.com/linxGnu/grocksdb v1.7.15
@@ -28,7 +28,7 @@ require (
2828
golang.org/x/exp v0.0.0-20230206171751-46f607a40771
2929
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4
3030
google.golang.org/grpc v1.53.0
31-
google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8
31+
google.golang.org/protobuf v1.29.1
3232
gopkg.in/yaml.v2 v2.4.0
3333
)
3434

@@ -68,7 +68,7 @@ require (
6868
github.com/cosmos/btcutil v1.0.5 // indirect
6969
github.com/cosmos/cosmos-proto v1.0.0-beta.1 // indirect
7070
github.com/cosmos/go-bip39 v1.0.0 // indirect
71-
github.com/cosmos/iavl v0.19.5 // indirect
71+
github.com/cosmos/iavl v0.19.6-0.20230410112350-e1e212a3a3aa // indirect
7272
github.com/cosmos/ledger-cosmos-go v0.12.2 // indirect
7373
github.com/creachadair/taskgroup v0.3.2 // indirect
7474
github.com/crypto-org-chain/cronos/memiavl v0.0.1 // indirect

‎go.sum‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -389,10 +389,10 @@ github.com/cosmos/cosmos-sdk v0.46.11/go.mod h1:bG4AkW9bqc8ycrryyKGQEl3YV9BY2wr6
389389
github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y=
390390
github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY=
391391
github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw=
392-
github.com/cosmos/gogoproto v1.4.2 h1:UeGRcmFW41l0G0MiefWhkPEVEwvu78SZsHBvI78dAYw=
393-
github.com/cosmos/gogoproto v1.4.2/go.mod h1:cLxOsn1ljAHSV527CHOtaIP91kK6cCrZETRBrkzItWU=
394-
github.com/cosmos/iavl v0.19.5 h1:rGA3hOrgNxgRM5wYcSCxgQBap7fW82WZgY78V9po/iY=
395-
github.com/cosmos/iavl v0.19.5/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw=
392+
github.com/cosmos/gogoproto v1.4.7 h1:RzYKVnsEC7UIkDnhTIkqEB7LnIQbsySvmNEqPCiPevk=
393+
github.com/cosmos/gogoproto v1.4.7/go.mod h1:gxGePp9qedovvl/StQL2BIJ6qlIBn1+9YxR0IulGBKA=
394+
github.com/cosmos/iavl v0.19.6-0.20230410112350-e1e212a3a3aa h1:nRlSq1UT5NSiTfcQMDCrxxexLt4+VrFNAVyMIUrb6ro=
395+
github.com/cosmos/iavl v0.19.6-0.20230410112350-e1e212a3a3aa/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw=
396396
github.com/cosmos/ibc-go/v5 v5.2.0 h1:LxwttRQqdUJpQ3/Gc3XPg5lkRo3pcbzx65dxFIY6ONE=
397397
github.com/cosmos/ibc-go/v5 v5.2.0/go.mod h1:MhDUMDVSboK5JW2pEWHNcw0wJHaHqKV/vwwP7awGhzI=
398398
github.com/cosmos/ledger-cosmos-go v0.12.2 h1:/XYaBlE2BJxtvpkHiBm97gFGSGmYGKunKyF3nNqAXZA=
@@ -600,8 +600,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
600600
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
601601
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
602602
github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
603-
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
604603
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
604+
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
605+
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
605606
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
606607
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
607608
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
@@ -1878,8 +1879,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
18781879
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
18791880
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
18801881
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
1881-
google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 h1:KR8+MyP7/qOlV+8Af01LtjL04bu7on42eVsxT4EyBQk=
1882-
google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
1882+
google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM=
1883+
google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
18831884
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
18841885
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
18851886
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

‎gomod2nix.toml‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -122,11 +122,11 @@ schema = 3
122122
version = "v1.0.0"
123123
hash = "sha256-Qm2aC2vaS8tjtMUbHmlBSagOSqbduEEDwc51qvQaBmA="
124124
[mod."github.com/cosmos/gogoproto"]
125-
version = "v1.4.2"
126-
hash = "sha256-hOY+mhPDYWcSYSdth2AW7IONdgicqQir0z/1XrXt9NY="
125+
version = "v1.4.7"
126+
hash = "sha256-JGSKV4CMgBGQYR7kZt6QQsVjgLEyAjNzKrtLalJqqVo="
127127
[mod."github.com/cosmos/iavl"]
128-
version = "v0.19.5"
129-
hash = "sha256-8PerCyxQrBCtr2zkhgriqauhCSoIe580dsYpZ44o+cE="
128+
version = "v0.19.6-0.20230410112350-e1e212a3a3aa"
129+
hash = "sha256-OF1PLzuetcucchOyE6spTVKNacgGxbFocz6+Qm+Q5DI="
130130
[mod."github.com/cosmos/ibc-go/v5"]
131131
version = "v5.2.0"
132132
hash = "sha256-Gfqhdz9ZKEgb7LCkHiXCwYZYUYluQ+vMew/GkssfVj8="
@@ -227,8 +227,8 @@ schema = 3
227227
version = "v0.0.0-20210331224755-41bb18bfe9da"
228228
hash = "sha256-7Gs7CS9gEYZkbu5P4hqPGBpeGZWC64VDwraSKFF+VR0="
229229
[mod."github.com/golang/protobuf"]
230-
version = "v1.5.2"
231-
hash = "sha256-IVwooaIo46iq7euSSVWTBAdKd+2DUaJ67MtBao1DpBI="
230+
version = "v1.5.3"
231+
hash = "sha256-svogITcP4orUIsJFjMtp+Uv1+fKJv2Q5Zwf2dMqnpOQ="
232232
[mod."github.com/golang/snappy"]
233233
version = "v0.0.4"
234234
hash = "sha256-Umx+5xHAQCN/Gi4HbtMhnDCSPFAXSsjVbXd8n5LhjAA="
@@ -586,8 +586,8 @@ schema = 3
586586
version = "v1.53.0"
587587
hash = "sha256-LkB13k1JaQ7e4nGpCoEA9q4T8oIV0KvkFIDZmHhDr08="
588588
[mod."google.golang.org/protobuf"]
589-
version = "v1.28.2-0.20220831092852-f930b1dc76e8"
590-
hash = "sha256-li5hXlXwTJ5LIZ8bVki1AZ6UFI2gXHl33JwdX1dOrtM="
589+
version = "v1.29.1"
590+
hash = "sha256-ilSVvttGSP2xpqpoyQ0/Iuyx1WMiwe6GASKTfoeaqxw="
591591
[mod."gopkg.in/ini.v1"]
592592
version = "v1.67.0"
593593
hash = "sha256-V10ahGNGT+NLRdKUyRg1dos5RxLBXBk1xutcnquc/+4="

‎memiavl/benchmark_test.go‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,28 +19,24 @@ func BenchmarkByteCompare(b *testing.B) {
1919
}
2020

2121
func BenchmarkRandomGet(b *testing.B) {
22-
b.Cleanup(removeDefaultWal)
23-
2422
amount := 1000000
2523
items := genRandItems(amount)
2624
targetKey := items[500].key
2725
targetValue := items[500].value
2826
targetItem := itemT{key: targetKey}
2927

30-
tree, err := New(DefaultPathToWAL)
31-
require.NoError(b, err)
28+
tree := New()
3229
for _, item := range items {
3330
tree.set(item.key, item.value)
3431
}
3532

3633
snapshotDir := b.TempDir()
37-
err = tree.WriteSnapshot(snapshotDir, true)
34+
err := tree.WriteSnapshot(snapshotDir, true)
3835
require.NoError(b, err)
3936
snapshot, err := OpenSnapshot(snapshotDir)
4037
require.NoError(b, err)
4138
defer snapshot.Close()
42-
diskTree, err := NewFromSnapshot(snapshot, DefaultPathToWAL)
43-
require.NoError(b, err)
39+
diskTree := NewFromSnapshot(snapshot)
4440

4541
require.Equal(b, targetValue, tree.Get(targetKey))
4642
require.Equal(b, targetValue, diskTree.Get(targetKey))
@@ -124,13 +120,11 @@ func BenchmarkRandomGet(b *testing.B) {
124120
}
125121

126122
func BenchmarkRandomSet(b *testing.B) {
127-
b.Cleanup(removeDefaultWal)
128123
items := genRandItems(1000000)
129124
b.ResetTimer()
130125
b.Run("memiavl", func(b *testing.B) {
131126
for i := 0; i < b.N; i++ {
132-
tree, err := New(DefaultPathToWAL)
133-
require.NoError(b, err)
127+
tree := New()
134128
for _, item := range items {
135129
tree.set(item.key, item.value)
136130
}

‎memiavl/changeset.go‎

Lines changed: 0 additions & 130 deletions
This file was deleted.

0 commit comments

Comments
 (0)