Merkledb Attempt to reduce test runtime

x/merkledb/codec_test.go

@@ -28,7 +28,7 @@ func FuzzCodecBool(f *testing.F) {
 			startLen := reader.Len()
 			got, err := codec.decodeBool(reader)
 			if err != nil {
-				return
+				t.SkipNow()
 			}
 			endLen := reader.Len()
 			numRead := startLen - endLen
@@ -56,7 +56,7 @@ func FuzzCodecInt(f *testing.F) {
 			startLen := reader.Len()
 			got, err := codec.decodeInt(reader)
 			if err != nil {
-				return
+				t.SkipNow()
 			}
 			endLen := reader.Len()
 			numRead := startLen - endLen
@@ -84,7 +84,7 @@ func FuzzCodecSerializedPath(f *testing.F) {
 			startLen := reader.Len()
 			got, err := codec.decodeSerializedPath(reader)
 			if err != nil {
-				return
+				t.SkipNow()
 			}
 			endLen := reader.Len()
 			numRead := startLen - endLen
@@ -113,7 +113,7 @@ func FuzzCodecDBNodeCanonical(f *testing.F) {
 			codec := codec.(*codecImpl)
 			node := &dbNode{}
 			if err := codec.decodeDBNode(b, node); err != nil {
-				return
+				t.SkipNow()
 			}
 
 			// Encoding [node] should be the same as [b].

x/merkledb/db_test.go

@@ -160,7 +160,7 @@ func Test_MerkleDB_DB_Load_Root_From_DB(t *testing.T) {
 func Test_MerkleDB_DB_Rebuild(t *testing.T) {
 	require := require.New(t)
 
-	initialSize := 10_000
+	initialSize := 5_000
 
 	config := newDefaultConfig()
 	config.ValueNodeCacheSize = uint(initialSize)
@@ -726,36 +726,36 @@ func Test_MerkleDB_Random_Insert_Ordering(t *testing.T) {
 	}
 }
 
-func Test_MerkleDB_RandomCases(t *testing.T) {
-	require := require.New(t)
-
-	const (
-		minSize              = 150
-		maxSize              = 500
-		checkHashProbability = 0.01
-	)
-
-	for size := minSize; size < maxSize; size += 10 {
-		now := time.Now().UnixNano()
-		t.Logf("seed for iter %d: %d", size, now)
-		r := rand.New(rand.NewSource(now)) // #nosec G404
-		runRandDBTest(require, r, generateRandTest(require, r, size, checkHashProbability))
-	}
+func FuzzMerkleDBEmptyRandomizedActions(f *testing.F) {
+	f.Fuzz(
+		func(
+			t *testing.T,
+			randSeed int64,
+			size uint,
+		) {
+			if size == 0 {
+				t.SkipNow()
+			}
+			require := require.New(t)
+			r := rand.New(rand.NewSource(randSeed)) // #nosec G404
+			runRandDBTest(require, r, generateRandTest(require, r, size, 0.01 /*checkHashProbability*/))
+		})
 }
 
-func Test_MerkleDB_RandomCases_InitialValues(t *testing.T) {
-	require := require.New(t)
-
-	const (
-		initialValues        = 1_000
-		updates              = 2_500
-		checkHashProbability = 0
-	)
-
-	now := time.Now().UnixNano()
-	t.Logf("seed: %d", now)
-	r := rand.New(rand.NewSource(now)) // #nosec G404
-	runRandDBTest(require, r, generateInitialValues(require, r, initialValues, updates, checkHashProbability))
+func FuzzMerkleDBInitialValuesRandomizedActions(f *testing.F) {
+	f.Fuzz(func(
+		t *testing.T,
+		initialValues uint,
+		numSteps uint,
+		randSeed int64,
+	) {
+		if numSteps == 0 {
+			t.SkipNow()
+		}
+		require := require.New(t)
+		r := rand.New(rand.NewSource(randSeed)) // #nosec G404
+		runRandDBTest(require, r, generateInitialValues(require, r, initialValues, numSteps, 0.001 /*checkHashProbability*/))
+	})
 }
 
 // randTest performs random trie operations.
@@ -959,7 +959,7 @@ func generateRandTestWithKeys(
 	require *require.Assertions,
 	r *rand.Rand,
 	allKeys [][]byte,
-	size int,
+	size uint,
 	checkHashProbability float64,
 ) randTest {
 	const nilEndProbability = 0.1
@@ -1007,7 +1007,7 @@ func generateRandTestWithKeys(
 	}
 
 	var steps randTest
-	for i := 0; i < size-1; {
+	for i := uint(0); i < size-1; {
 		step := randTestStep{op: r.Intn(opMax)}
 		switch step.op {
 		case opUpdate:
@@ -1041,8 +1041,8 @@ func generateRandTestWithKeys(
 func generateInitialValues(
 	require *require.Assertions,
 	r *rand.Rand,
-	numInitialKeyValues int,
-	size int,
+	numInitialKeyValues uint,
+	size uint,
 	percentChanceToFullHash float64,
 ) randTest {
 	const (
@@ -1070,7 +1070,7 @@ func generateInitialValues(
 	}
 
 	var steps randTest
-	for i := 0; i < numInitialKeyValues; i++ {
+	for i := uint(0); i < numInitialKeyValues; i++ {
 		step := randTestStep{
 			op:    opUpdate,
 			key:   genKey(),
@@ -1091,7 +1091,7 @@ func generateInitialValues(
 	return steps
 }
 
-func generateRandTest(require *require.Assertions, r *rand.Rand, size int, percentChanceToFullHash float64) randTest {
+func generateRandTest(require *require.Assertions, r *rand.Rand, size uint, percentChanceToFullHash float64) randTest {
 	return generateRandTestWithKeys(require, r, [][]byte{}, size, percentChanceToFullHash)
 }
 
@@ -1101,15 +1101,15 @@ func insertRandomKeyValues(
 	require *require.Assertions,
 	rand *rand.Rand,
 	dbs []database.Database,
-	numKeyValues int,
+	numKeyValues uint,
 	deletePortion float64,
 ) {
 	maxKeyLen := units.KiB
 	maxValLen := 4 * units.KiB
 
 	require.GreaterOrEqual(deletePortion, float64(0))
 	require.LessOrEqual(deletePortion, float64(1))
-	for i := 0; i < numKeyValues; i++ {
+	for i := uint(0); i < numKeyValues; i++ {
 		keyLen := rand.Intn(maxKeyLen)
 		key := make([]byte, keyLen)
 		_, _ = rand.Read(key)

x/merkledb/helpers_test.go

@@ -0,0 +1,79 @@
+// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package merkledb
+
+import (
+	"context"
+	"math/rand"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ava-labs/avalanchego/database/memdb"
+	"github.com/ava-labs/avalanchego/ids"
+	"github.com/ava-labs/avalanchego/utils/hashing"
+	"github.com/ava-labs/avalanchego/utils/maybe"
+)
+
+func getBasicDB() (*merkleDB, error) {
+	return newDatabase(
+		context.Background(),
+		memdb.New(),
+		newDefaultConfig(),
+		&mockMetrics{},
+	)
+}
+
+// Writes []byte{i} -> []byte{i} for i in [0, 4]
+func writeBasicBatch(t *testing.T, db *merkleDB) {
+	require := require.New(t)
+
+	batch := db.NewBatch()
+	require.NoError(batch.Put([]byte{0}, []byte{0}))
+	require.NoError(batch.Put([]byte{1}, []byte{1}))
+	require.NoError(batch.Put([]byte{2}, []byte{2}))
+	require.NoError(batch.Put([]byte{3}, []byte{3}))
+	require.NoError(batch.Put([]byte{4}, []byte{4}))
+	require.NoError(batch.Write())
+}
+
+func newRandomProofNode(r *rand.Rand) ProofNode {
+	key := make([]byte, r.Intn(32)) // #nosec G404
+	_, _ = r.Read(key)              // #nosec G404
+	serializedKey := newPath(key).Serialize()
+
+	val := make([]byte, r.Intn(64)) // #nosec G404
+	_, _ = r.Read(val)              // #nosec G404
+
+	children := map[byte]ids.ID{}
+	for j := 0; j < NodeBranchFactor; j++ {
+		if r.Float64() < 0.5 {
+			var childID ids.ID
+			_, _ = r.Read(childID[:]) // #nosec G404
+			children[byte(j)] = childID
+		}
+	}
+
+	hasValue := rand.Intn(2) == 1 // #nosec G404
+	var valueOrHash maybe.Maybe[[]byte]
+	if hasValue {
+		// use the hash instead when length is greater than the hash length
+		if len(val) >= HashLength {
+			val = hashing.ComputeHash256(val)
+		} else if len(val) == 0 {
+			// We do this because when we encode a value of []byte{} we will later
+			// decode it as nil.
+			// Doing this prevents inconsistency when comparing the encoded and
+			// decoded values.
+			val = nil
+		}
+		valueOrHash = maybe.Some(val)
+	}
+
+	return ProofNode{
+		KeyPath:     serializedKey,
+		ValueOrHash: valueOrHash,
+		Children:    children,
+	}
+}

x/merkledb/history_test.go

@@ -86,12 +86,11 @@ func Test_History_Large(t *testing.T) {
 
 	numIters := 250
 
-	for i := 1; i < 10; i++ {
+	for i := 1; i < 5; i++ {
 		config := newDefaultConfig()
 		// History must be large enough to get the change proof
-		// after this loop. Multiply by four because every loop
-		// iteration we do two puts and up to two deletes.
-		config.HistoryLength = uint(4 * numIters)
+		// after this loop.
+		config.HistoryLength = uint(numIters)
 		db, err := New(
 			context.Background(),
 			memdb.New(),
@@ -105,40 +104,45 @@ func Test_History_Large(t *testing.T) {
 		r := rand.New(rand.NewSource(now)) // #nosec G404
 		// make sure they stay in sync
 		for x := 0; x < numIters; x++ {
+			batch := db.NewBatch()
 			addkey := make([]byte, r.Intn(50))
 			_, err := r.Read(addkey)
 			require.NoError(err)
 			val := make([]byte, r.Intn(50))
 			_, err = r.Read(val)
 			require.NoError(err)
 
-			require.NoError(db.Put(addkey, val))
+			require.NoError(batch.Put(addkey, val))
 
 			addNilkey := make([]byte, r.Intn(50))
 			_, err = r.Read(addNilkey)
 			require.NoError(err)
-			require.NoError(db.Put(addNilkey, nil))
+			require.NoError(batch.Put(addNilkey, nil))
 
 			deleteKeyStart := make([]byte, r.Intn(50))
 			_, err = r.Read(deleteKeyStart)
 			require.NoError(err)
 
 			it := db.NewIteratorWithStart(deleteKeyStart)
 			if it.Next() {
-				require.NoError(db.Delete(it.Key()))
+				require.NoError(batch.Delete(it.Key()))
 			}
 			require.NoError(it.Error())
 			it.Release()
 
+			require.NoError(batch.Write())
 			root, err := db.GetMerkleRoot(context.Background())
 			require.NoError(err)
 			roots = append(roots, root)
 		}
-		proof, err := db.GetRangeProofAtRoot(context.Background(), roots[0], maybe.Nothing[[]byte](), maybe.Nothing[[]byte](), 10)
-		require.NoError(err)
-		require.NotNil(proof)
 
-		require.NoError(proof.Verify(context.Background(), maybe.Nothing[[]byte](), maybe.Nothing[[]byte](), roots[0]))
+		for i := 0; i < numIters; i += numIters / 10 {
+			proof, err := db.GetRangeProofAtRoot(context.Background(), roots[i], maybe.Nothing[[]byte](), maybe.Nothing[[]byte](), 10)
+			require.NoError(err)
+			require.NotNil(proof)
+
+			require.NoError(proof.Verify(context.Background(), maybe.Nothing[[]byte](), maybe.Nothing[[]byte](), roots[i]))
+		}
 	}
 }
 

x/merkledb/proof.go

@@ -70,6 +70,10 @@ func (node *ProofNode) ToProto() *pb.ProofNode {
 			NibbleLength: uint64(node.KeyPath.NibbleLength),
 			Value:        node.KeyPath.Value,
 		},
+		ValueOrHash: &pb.MaybeBytes{
+			Value:     node.ValueOrHash.Value(),
+			IsNothing: node.ValueOrHash.IsNothing(),
+		},
 		Children: make(map[uint32][]byte, len(node.Children)),
 	}
 
@@ -78,16 +82,6 @@ func (node *ProofNode) ToProto() *pb.ProofNode {
 		pbNode.Children[uint32(childIndex)] = childID[:]
 	}
 
-	if node.ValueOrHash.HasValue() {
-		pbNode.ValueOrHash = &pb.MaybeBytes{
-			Value: node.ValueOrHash.Value(),
-		}
-	} else {
-		pbNode.ValueOrHash = &pb.MaybeBytes{
-			IsNothing: true,
-		}
-	}
-
 	return pbNode
 }
 
@@ -568,21 +562,12 @@ func (proof *ChangeProof) ToProto() *pb.ChangeProof {
 
 	keyChanges := make([]*pb.KeyChange, len(proof.KeyChanges))
 	for i, kv := range proof.KeyChanges {
-		var value pb.MaybeBytes
-		if kv.Value.HasValue() {
-			value = pb.MaybeBytes{
-				Value:     kv.Value.Value(),
-				IsNothing: false,
-			}
-		} else {
-			value = pb.MaybeBytes{
-				Value:     nil,
-				IsNothing: true,
-			}
-		}
 		keyChanges[i] = &pb.KeyChange{
-			Key:   kv.Key,
-			Value: &value,
+			Key: kv.Key,
+			Value: &pb.MaybeBytes{
+				Value:     kv.Value.Value(),
+				IsNothing: kv.Value.IsNothing(),
+			},
 		}
 	}