Cache LevelDB snapshots and use lock-free map (#490)

- Replacing map with sync.Map
- Caching DB snapshot after commit to avoid creating a snapshot for each
Get operation
- Modifying cache to return byte array to avoid code duplication
- Remove Db lock because levelDB is already protected
- Reduce code duplication in Open()

Signed-off-by: Liran Funaro <liran.funaro@gmail.com>

---------

Signed-off-by: Liran Funaro <liran.funaro@gmail.com>

Author: liran-funaro

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2
 	go.etcd.io/etcd v0.5.0-alpha.5.0.20210226220824-aa7126864d82 // indirect git tag v3.4.15
+	go.uber.org/multierr v1.6.0
 	go.uber.org/zap v1.18.1
 	google.golang.org/protobuf v1.26.0
 	gopkg.in/yaml.v2 v2.4.0

internal/worldstate/leveldb/cache.go

@@ -2,8 +2,6 @@ package leveldb
 
 import (
 	"github.com/VictoriaMetrics/fastcache"
-	"github.com/golang/protobuf/proto"
-	"github.com/hyperledger-labs/orion-server/pkg/types"
 )
 
 var keySep = []byte{0x00}
@@ -27,19 +25,9 @@ func newCache(dataCacheSizeMBs int) *cache {
 
 // getState returns the value for a given namespace and key from
 // the cache
-func (c *cache) getState(namespace, key string) (*types.ValueWithMetadata, error) {
+func (c *cache) getState(namespace, key string) ([]byte, bool) {
 	cacheKey := constructCacheKey(namespace, key)
-
-	valBytes, exists := c.dataCache.HasGet(nil, cacheKey)
-	if !exists {
-		return nil, nil
-	}
-
-	cacheValue := &types.ValueWithMetadata{}
-	if err := proto.Unmarshal(valBytes, cacheValue); err != nil {
-		return nil, err
-	}
-	return cacheValue, nil
+	return c.dataCache.HasGet(nil, cacheKey)
 }
 
 // putState stores a given value in the cache

internal/worldstate/leveldb/cache_test.go

@@ -9,6 +9,17 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
+func unmarshaler(t *testing.T) func(b []byte, inCache bool) (*types.ValueWithMetadata, bool) {
+	return func(b []byte, inCache bool) (*types.ValueWithMetadata, bool) {
+		if !inCache {
+			return nil, inCache
+		}
+		value := &types.ValueWithMetadata{}
+		require.NoError(t, proto.Unmarshal(b, value))
+		return value, inCache
+	}
+}
+
 func TestCache(t *testing.T) {
 	cache := newCache(10)
 
@@ -34,8 +45,8 @@ func TestCache(t *testing.T) {
 
 	t.Run("check basic storage and retrieval", func(t *testing.T) {
 		// db1, key1 does not exist
-		v, err := cache.getState("db1", "key1")
-		require.NoError(t, err)
+		v, inCache := cache.getState("db1", "key1")
+		require.False(t, inCache)
 		require.Nil(t, v)
 
 		s := &fastcache.Stats{}
@@ -54,11 +65,12 @@ func TestCache(t *testing.T) {
 		}
 		valBytes, err := proto.Marshal(db1Key1Value1)
 		require.NoError(t, err)
-		cache.putState("db1", "key1", valBytes)
+		require.NoError(t, cache.putState("db1", "key1", valBytes))
 
-		// db1, key1 should exists
-		actualKey1Value1, err := cache.getState("db1", "key1")
-		require.NoError(t, err)
+		// db1, key1 should exist
+		actualKey1Value1, inCache := unmarshaler(t)(cache.getState("db1", "key1"))
+		require.True(t, inCache)
+		require.NotNil(t, actualKey1Value1)
 		require.True(t, proto.Equal(db1Key1Value1, actualKey1Value1))
 
 		cache.dataCache.UpdateStats(s)
@@ -69,11 +81,12 @@ func TestCache(t *testing.T) {
 		// update key1's value
 		valBytes, err := proto.Marshal(db1Key1Value2)
 		require.NoError(t, err)
-		cache.putState("db1", "key1", valBytes)
+		require.NoError(t, cache.putState("db1", "key1", valBytes))
 
 		// db1, key1 should have the updated value
-		actualKey1Value2, err := cache.getState("db1", "key1")
-		require.NoError(t, err)
+		actualKey1Value2, inCache := unmarshaler(t)(cache.getState("db1", "key1"))
+		require.True(t, inCache)
+		require.NotNil(t, actualKey1Value2)
 		require.True(t, proto.Equal(db1Key1Value2, actualKey1Value2))
 
 		s := &fastcache.Stats{}
@@ -87,8 +100,8 @@ func TestCache(t *testing.T) {
 		require.NoError(t, err)
 		cache.putStateIfExist("db2", "key1", valBytes)
 
-		v, err := cache.getState("db2", "key1")
-		require.NoError(t, err)
+		v, inCache := unmarshaler(t)(cache.getState("db2", "key1"))
+		require.False(t, inCache)
 		require.Nil(t, v)
 
 		s := &fastcache.Stats{}
@@ -99,15 +112,17 @@ func TestCache(t *testing.T) {
 	t.Run("store same key in two different databases", func(t *testing.T) {
 		valBytes, err := proto.Marshal(db2Key1Value1)
 		require.NoError(t, err)
-		cache.putState("db2", "key1", valBytes)
+		require.NoError(t, cache.putState("db2", "key1", valBytes))
 
 		// both db1, key1 and db2, key1 should exist
-		actualDB1Key1Value2, err := cache.getState("db1", "key1")
-		require.NoError(t, err)
+		actualDB1Key1Value2, inCache := unmarshaler(t)(cache.getState("db1", "key1"))
+		require.True(t, inCache)
+		require.NotNil(t, actualDB1Key1Value2)
 		require.True(t, proto.Equal(db1Key1Value2, actualDB1Key1Value2))
 
-		actualDB2Key1Value1, err := cache.getState("db2", "key1")
-		require.NoError(t, err)
+		actualDB2Key1Value1, inCache := unmarshaler(t)(cache.getState("db2", "key1"))
+		require.True(t, inCache)
+		require.NotNil(t, actualDB2Key1Value1)
 		require.True(t, proto.Equal(db2Key1Value1, actualDB2Key1Value1))
 
 		s := &fastcache.Stats{}