Skip to content

Commit 05af816

Browse files
dboehm-avalabsDan Lainemaru-ava
authored
Merkledb Attempt to reduce test runtime (#1990)
Signed-off-by: David Boehm <91908103+dboehm-avalabs@users.noreply.github.com> Co-authored-by: Dan Laine <daniel.laine@avalabs.org> Co-authored-by: marun <maru.newby@avalabs.org>
1 parent e58b3e0 commit 05af816

File tree

6 files changed

+238
-263
lines changed

6 files changed

+238
-263
lines changed

‎x/merkledb/codec_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ func FuzzCodecBool(f *testing.F) {
2828
startLen := reader.Len()
2929
got, err := codec.decodeBool(reader)
3030
if err != nil {
31-
return
31+
t.SkipNow()
3232
}
3333
endLen := reader.Len()
3434
numRead := startLen - endLen
@@ -56,7 +56,7 @@ func FuzzCodecInt(f *testing.F) {
5656
startLen := reader.Len()
5757
got, err := codec.decodeInt(reader)
5858
if err != nil {
59-
return
59+
t.SkipNow()
6060
}
6161
endLen := reader.Len()
6262
numRead := startLen - endLen
@@ -84,7 +84,7 @@ func FuzzCodecSerializedPath(f *testing.F) {
8484
startLen := reader.Len()
8585
got, err := codec.decodeSerializedPath(reader)
8686
if err != nil {
87-
return
87+
t.SkipNow()
8888
}
8989
endLen := reader.Len()
9090
numRead := startLen - endLen
@@ -113,7 +113,7 @@ func FuzzCodecDBNodeCanonical(f *testing.F) {
113113
codec := codec.(*codecImpl)
114114
node := &dbNode{}
115115
if err := codec.decodeDBNode(b, node); err != nil {
116-
return
116+
t.SkipNow()
117117
}
118118

119119
// Encoding [node] should be the same as [b].

‎x/merkledb/db_test.go‎

Lines changed: 37 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ func Test_MerkleDB_DB_Load_Root_From_DB(t *testing.T) {
160160
func Test_MerkleDB_DB_Rebuild(t *testing.T) {
161161
require := require.New(t)
162162

163-
initialSize := 10_000
163+
initialSize := 5_000
164164

165165
config := newDefaultConfig()
166166
config.ValueNodeCacheSize = uint(initialSize)
@@ -726,36 +726,36 @@ func Test_MerkleDB_Random_Insert_Ordering(t *testing.T) {
726726
}
727727
}
728728

729-
func Test_MerkleDB_RandomCases(t *testing.T) {
730-
require := require.New(t)
731-
732-
const (
733-
minSize = 150
734-
maxSize = 500
735-
checkHashProbability = 0.01
736-
)
737-
738-
for size := minSize; size < maxSize; size += 10 {
739-
now := time.Now().UnixNano()
740-
t.Logf("seed for iter %d: %d", size, now)
741-
r := rand.New(rand.NewSource(now)) // #nosec G404
742-
runRandDBTest(require, r, generateRandTest(require, r, size, checkHashProbability))
743-
}
729+
func FuzzMerkleDBEmptyRandomizedActions(f *testing.F) {
730+
f.Fuzz(
731+
func(
732+
t *testing.T,
733+
randSeed int64,
734+
size uint,
735+
) {
736+
if size == 0 {
737+
t.SkipNow()
738+
}
739+
require := require.New(t)
740+
r := rand.New(rand.NewSource(randSeed)) // #nosec G404
741+
runRandDBTest(require, r, generateRandTest(require, r, size, 0.01 /*checkHashProbability*/))
742+
})
744743
}
745744

746-
func Test_MerkleDB_RandomCases_InitialValues(t *testing.T) {
747-
require := require.New(t)
748-
749-
const (
750-
initialValues = 1_000
751-
updates = 2_500
752-
checkHashProbability = 0
753-
)
754-
755-
now := time.Now().UnixNano()
756-
t.Logf("seed: %d", now)
757-
r := rand.New(rand.NewSource(now)) // #nosec G404
758-
runRandDBTest(require, r, generateInitialValues(require, r, initialValues, updates, checkHashProbability))
745+
func FuzzMerkleDBInitialValuesRandomizedActions(f *testing.F) {
746+
f.Fuzz(func(
747+
t *testing.T,
748+
initialValues uint,
749+
numSteps uint,
750+
randSeed int64,
751+
) {
752+
if numSteps == 0 {
753+
t.SkipNow()
754+
}
755+
require := require.New(t)
756+
r := rand.New(rand.NewSource(randSeed)) // #nosec G404
757+
runRandDBTest(require, r, generateInitialValues(require, r, initialValues, numSteps, 0.001 /*checkHashProbability*/))
758+
})
759759
}
760760

761761
// randTest performs random trie operations.
@@ -959,7 +959,7 @@ func generateRandTestWithKeys(
959959
require *require.Assertions,
960960
r *rand.Rand,
961961
allKeys [][]byte,
962-
size int,
962+
size uint,
963963
checkHashProbability float64,
964964
) randTest {
965965
const nilEndProbability = 0.1
@@ -1007,7 +1007,7 @@ func generateRandTestWithKeys(
10071007
}
10081008

10091009
var steps randTest
1010-
for i := 0; i < size-1; {
1010+
for i := uint(0); i < size-1; {
10111011
step := randTestStep{op: r.Intn(opMax)}
10121012
switch step.op {
10131013
case opUpdate:
@@ -1041,8 +1041,8 @@ func generateRandTestWithKeys(
10411041
func generateInitialValues(
10421042
require *require.Assertions,
10431043
r *rand.Rand,
1044-
numInitialKeyValues int,
1045-
size int,
1044+
numInitialKeyValues uint,
1045+
size uint,
10461046
percentChanceToFullHash float64,
10471047
) randTest {
10481048
const (
@@ -1070,7 +1070,7 @@ func generateInitialValues(
10701070
}
10711071

10721072
var steps randTest
1073-
for i := 0; i < numInitialKeyValues; i++ {
1073+
for i := uint(0); i < numInitialKeyValues; i++ {
10741074
step := randTestStep{
10751075
op: opUpdate,
10761076
key: genKey(),
@@ -1091,7 +1091,7 @@ func generateInitialValues(
10911091
return steps
10921092
}
10931093

1094-
func generateRandTest(require *require.Assertions, r *rand.Rand, size int, percentChanceToFullHash float64) randTest {
1094+
func generateRandTest(require *require.Assertions, r *rand.Rand, size uint, percentChanceToFullHash float64) randTest {
10951095
return generateRandTestWithKeys(require, r, [][]byte{}, size, percentChanceToFullHash)
10961096
}
10971097

@@ -1101,15 +1101,15 @@ func insertRandomKeyValues(
11011101
require *require.Assertions,
11021102
rand *rand.Rand,
11031103
dbs []database.Database,
1104-
numKeyValues int,
1104+
numKeyValues uint,
11051105
deletePortion float64,
11061106
) {
11071107
maxKeyLen := units.KiB
11081108
maxValLen := 4 * units.KiB
11091109

11101110
require.GreaterOrEqual(deletePortion, float64(0))
11111111
require.LessOrEqual(deletePortion, float64(1))
1112-
for i := 0; i < numKeyValues; i++ {
1112+
for i := uint(0); i < numKeyValues; i++ {
11131113
keyLen := rand.Intn(maxKeyLen)
11141114
key := make([]byte, keyLen)
11151115
_, _ = rand.Read(key)

‎x/merkledb/helpers_test.go‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
2+
// See the file LICENSE for licensing terms.
3+
4+
package merkledb
5+
6+
import (
7+
"context"
8+
"math/rand"
9+
"testing"
10+
11+
"github.com/stretchr/testify/require"
12+
13+
"github.com/ava-labs/avalanchego/database/memdb"
14+
"github.com/ava-labs/avalanchego/ids"
15+
"github.com/ava-labs/avalanchego/utils/hashing"
16+
"github.com/ava-labs/avalanchego/utils/maybe"
17+
)
18+
19+
func getBasicDB() (*merkleDB, error) {
20+
return newDatabase(
21+
context.Background(),
22+
memdb.New(),
23+
newDefaultConfig(),
24+
&mockMetrics{},
25+
)
26+
}
27+
28+
// Writes []byte{i} -> []byte{i} for i in [0, 4]
29+
func writeBasicBatch(t *testing.T, db *merkleDB) {
30+
require := require.New(t)
31+
32+
batch := db.NewBatch()
33+
require.NoError(batch.Put([]byte{0}, []byte{0}))
34+
require.NoError(batch.Put([]byte{1}, []byte{1}))
35+
require.NoError(batch.Put([]byte{2}, []byte{2}))
36+
require.NoError(batch.Put([]byte{3}, []byte{3}))
37+
require.NoError(batch.Put([]byte{4}, []byte{4}))
38+
require.NoError(batch.Write())
39+
}
40+
41+
func newRandomProofNode(r *rand.Rand) ProofNode {
42+
key := make([]byte, r.Intn(32)) // #nosec G404
43+
_, _ = r.Read(key) // #nosec G404
44+
serializedKey := newPath(key).Serialize()
45+
46+
val := make([]byte, r.Intn(64)) // #nosec G404
47+
_, _ = r.Read(val) // #nosec G404
48+
49+
children := map[byte]ids.ID{}
50+
for j := 0; j < NodeBranchFactor; j++ {
51+
if r.Float64() < 0.5 {
52+
var childID ids.ID
53+
_, _ = r.Read(childID[:]) // #nosec G404
54+
children[byte(j)] = childID
55+
}
56+
}
57+
58+
hasValue := rand.Intn(2) == 1 // #nosec G404
59+
var valueOrHash maybe.Maybe[[]byte]
60+
if hasValue {
61+
// use the hash instead when length is greater than the hash length
62+
if len(val) >= HashLength {
63+
val = hashing.ComputeHash256(val)
64+
} else if len(val) == 0 {
65+
// We do this because when we encode a value of []byte{} we will later
66+
// decode it as nil.
67+
// Doing this prevents inconsistency when comparing the encoded and
68+
// decoded values.
69+
val = nil
70+
}
71+
valueOrHash = maybe.Some(val)
72+
}
73+
74+
return ProofNode{
75+
KeyPath: serializedKey,
76+
ValueOrHash: valueOrHash,
77+
Children: children,
78+
}
79+
}

‎x/merkledb/history_test.go‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -86,12 +86,11 @@ func Test_History_Large(t *testing.T) {
8686

8787
numIters := 250
8888

89-
for i := 1; i < 10; i++ {
89+
for i := 1; i < 5; i++ {
9090
config := newDefaultConfig()
9191
// History must be large enough to get the change proof
92-
// after this loop. Multiply by four because every loop
93-
// iteration we do two puts and up to two deletes.
94-
config.HistoryLength = uint(4 * numIters)
92+
// after this loop.
93+
config.HistoryLength = uint(numIters)
9594
db, err := New(
9695
context.Background(),
9796
memdb.New(),
@@ -105,40 +104,45 @@ func Test_History_Large(t *testing.T) {
105104
r := rand.New(rand.NewSource(now)) // #nosec G404
106105
// make sure they stay in sync
107106
for x := 0; x < numIters; x++ {
107+
batch := db.NewBatch()
108108
addkey := make([]byte, r.Intn(50))
109109
_, err := r.Read(addkey)
110110
require.NoError(err)
111111
val := make([]byte, r.Intn(50))
112112
_, err = r.Read(val)
113113
require.NoError(err)
114114

115-
require.NoError(db.Put(addkey, val))
115+
require.NoError(batch.Put(addkey, val))
116116

117117
addNilkey := make([]byte, r.Intn(50))
118118
_, err = r.Read(addNilkey)
119119
require.NoError(err)
120-
require.NoError(db.Put(addNilkey, nil))
120+
require.NoError(batch.Put(addNilkey, nil))
121121

122122
deleteKeyStart := make([]byte, r.Intn(50))
123123
_, err = r.Read(deleteKeyStart)
124124
require.NoError(err)
125125

126126
it := db.NewIteratorWithStart(deleteKeyStart)
127127
if it.Next() {
128-
require.NoError(db.Delete(it.Key()))
128+
require.NoError(batch.Delete(it.Key()))
129129
}
130130
require.NoError(it.Error())
131131
it.Release()
132132

133+
require.NoError(batch.Write())
133134
root, err := db.GetMerkleRoot(context.Background())
134135
require.NoError(err)
135136
roots = append(roots, root)
136137
}
137-
proof, err := db.GetRangeProofAtRoot(context.Background(), roots[0], maybe.Nothing[[]byte](), maybe.Nothing[[]byte](), 10)
138-
require.NoError(err)
139-
require.NotNil(proof)
140138

141-
require.NoError(proof.Verify(context.Background(), maybe.Nothing[[]byte](), maybe.Nothing[[]byte](), roots[0]))
139+
for i := 0; i < numIters; i += numIters / 10 {
140+
proof, err := db.GetRangeProofAtRoot(context.Background(), roots[i], maybe.Nothing[[]byte](), maybe.Nothing[[]byte](), 10)
141+
require.NoError(err)
142+
require.NotNil(proof)
143+
144+
require.NoError(proof.Verify(context.Background(), maybe.Nothing[[]byte](), maybe.Nothing[[]byte](), roots[i]))
145+
}
142146
}
143147
}
144148

‎x/merkledb/proof.go‎

Lines changed: 9 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,10 @@ func (node *ProofNode) ToProto() *pb.ProofNode {
7070
NibbleLength: uint64(node.KeyPath.NibbleLength),
7171
Value: node.KeyPath.Value,
7272
},
73+
ValueOrHash: &pb.MaybeBytes{
74+
Value: node.ValueOrHash.Value(),
75+
IsNothing: node.ValueOrHash.IsNothing(),
76+
},
7377
Children: make(map[uint32][]byte, len(node.Children)),
7478
}
7579

@@ -78,16 +82,6 @@ func (node *ProofNode) ToProto() *pb.ProofNode {
7882
pbNode.Children[uint32(childIndex)] = childID[:]
7983
}
8084

81-
if node.ValueOrHash.HasValue() {
82-
pbNode.ValueOrHash = &pb.MaybeBytes{
83-
Value: node.ValueOrHash.Value(),
84-
}
85-
} else {
86-
pbNode.ValueOrHash = &pb.MaybeBytes{
87-
IsNothing: true,
88-
}
89-
}
90-
9185
return pbNode
9286
}
9387

@@ -568,21 +562,12 @@ func (proof *ChangeProof) ToProto() *pb.ChangeProof {
568562

569563
keyChanges := make([]*pb.KeyChange, len(proof.KeyChanges))
570564
for i, kv := range proof.KeyChanges {
571-
var value pb.MaybeBytes
572-
if kv.Value.HasValue() {
573-
value = pb.MaybeBytes{
574-
Value: kv.Value.Value(),
575-
IsNothing: false,
576-
}
577-
} else {
578-
value = pb.MaybeBytes{
579-
Value: nil,
580-
IsNothing: true,
581-
}
582-
}
583565
keyChanges[i] = &pb.KeyChange{
584-
Key: kv.Key,
585-
Value: &value,
566+
Key: kv.Key,
567+
Value: &pb.MaybeBytes{
568+
Value: kv.Value.Value(),
569+
IsNothing: kv.Value.IsNothing(),
570+
},
586571
}
587572
}
588573

0 commit comments

Comments
 (0)