add pebble

badger_db.go

@@ -8,7 +8,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/dgraph-io/badger/v2"
+	"github.com/dgraph-io/badger/v3"
 )
 
 func init() { registerDBCreator(BadgerDBBackend, badgerDBCreator, true) }

db.go

@@ -33,8 +33,14 @@ const (
 	//   - requires gcc
 	//   - use rocksdb build tag (go build -tags rocksdb)
 	RocksDBBackend BackendType = "rocksdb"
-
+	// BadgerDBBackend represents badgerdb (uses github.com/dgraph-io/badger)
+	//   - EXPERIMENTAL
+	//   - use badgerdb build tag (go build -tags badgerdb)
 	BadgerDBBackend BackendType = "badgerdb"
+	// PebbleDBDBBackend represents pebble (uses github.com/cockroachdb/pebble)
+	//   - EXPERIMENTAL
+	//   - use pebble build tag (go build -tags pebbledb)
+	PebbleDBBackend BackendType = "pebbledb"
 )
 
 type dbCreator func(name string, dir string) (DB, error)

go.mod

@@ -3,29 +3,41 @@ module github.com/tendermint/tm-db
 go 1.17
 
 require (
-	github.com/dgraph-io/badger/v2 v2.2007.2
+	github.com/cockroachdb/pebble v0.0.0-20220419171548-8b3db52b6e3c
+	github.com/dgraph-io/badger/v3 v3.2103.2
 	github.com/gogo/protobuf v1.3.2
-	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/btree v1.0.0
 	github.com/jmhodges/levigo v1.0.0
+	github.com/linxGnu/grocksdb v1.7.1-0.20220327020612-ee0c6690507f
 	github.com/stretchr/testify v1.7.1
 	github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
 	go.etcd.io/bbolt v1.3.6
 	google.golang.org/grpc v1.45.0
 )
 
-require github.com/linxGnu/grocksdb v1.7.1-0.20220327020612-ee0c6690507f
-
 require (
-	github.com/DataDog/zstd v1.4.1 // indirect
+	github.com/DataDog/zstd v1.4.5 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/cockroachdb/errors v1.8.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
+	github.com/cockroachdb/redact v1.0.8 // indirect
+	github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect
-	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
+	github.com/dgraph-io/ristretto v0.1.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/golang/snappy v0.0.1 // indirect
-	github.com/pkg/errors v0.8.1 // indirect
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
+	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/flatbuffers v1.12.1 // indirect
+	github.com/klauspost/compress v1.12.3 // indirect
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/kr/text v0.1.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	go.opencensus.io v0.22.5 // indirect
+	golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect
 	golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect
 	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
 	golang.org/x/text v0.3.3 // indirect

makefile

@@ -28,9 +28,13 @@ test-badgerdb:
 	@echo "--> Running go test"
 	@go test $(PACKAGES) -tags badgerdb -v
 
+test-pebble:
+	@echo "--> Running go test"
+	@go test $(PACKAGES) -tags pebbledb -v
+
 test-all:
 	@echo "--> Running go test"
-	@go test $(PACKAGES) -tags cleveldb,boltdb,rocksdb,badgerdb -v
+	@go test $(PACKAGES) -tags cleveldb,boltdb,rocksdb,badgerdb,pebble -v
 
 lint:
 	@echo "--> Running linter"

pebble.go

@@ -0,0 +1,219 @@
+//go:build pebbledb
+
+package db
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/cockroachdb/pebble"
+)
+
+func init() {
+	dbCreator := func(name string, dir string) (DB, error) {
+		return NewPebbleDB(name, dir)
+	}
+	registerDBCreator(PebbleDBBackend, dbCreator, false)
+}
+
+// PebbleDB is a PebbleDB backend.
+type PebbleDB struct {
+	db *pebble.DB
+}
+
+var _ DB = (*PebbleDB)(nil)
+
+func NewPebbleDB(name string, dir string) (DB, error) {
+	dbPath := filepath.Join(dir, name+".db")
+	//	cache := pebble.NewCache(1024 * 1024 * 32)
+	//	defer cache.Unref()
+	opts := &pebble.Options{
+		//		Cache:                       cache,
+		//		FormatMajorVersion:          pebble.FormatNewest,
+		//		L0CompactionThreshold:       2,
+		//		L0StopWritesThreshold:       1000,
+		//		LBaseMaxBytes:               64 << 20, // 64 MB
+		//		Levels:                      make([]pebble.LevelOptions, 7),
+		//		MaxConcurrentCompactions:    3,
+		//		MaxOpenFiles:                1024,
+		//		MemTableSize:                64 << 20,
+		//		MemTableStopWritesThreshold: 4,
+	}
+	/*
+		for i := 0; i < len(opts.Levels); i++ {
+			l := &opts.Levels[i]
+			l.BlockSize = 32 << 10       // 32 KB
+			l.IndexBlockSize = 256 << 10 // 256 KB
+			l.FilterPolicy = bloom.FilterPolicy(10)
+			l.FilterType = pebble.TableFilter
+			if i > 0 {
+				l.TargetFileSize = opts.Levels[i-1].TargetFileSize * 2
+			}
+			l.EnsureDefaults()
+		}
+	*/
+	//	opts.Levels[6].FilterPolicy = nil
+	//	opts.FlushSplitBytes = opts.Levels[0].TargetFileSize
+
+	opts.EnsureDefaults()
+
+	p, err := pebble.Open(dbPath, opts)
+	if err != nil {
+		return nil, err
+	}
+	return &PebbleDB{
+		db: p,
+	}, err
+}
+
+// Get implements DB.
+func (db *PebbleDB) Get(key []byte) ([]byte, error) {
+	if len(key) == 0 {
+		return nil, errKeyEmpty
+	}
+	res, closer, err := db.db.Get(key)
+	if err != nil {
+		return res, nil
+	}
+	closer.Close()
+	return res, nil
+}
+
+// Has implements DB.
+func (db *PebbleDB) Has(key []byte) (bool, error) {
+	if len(key) == 0 {
+		return false, errKeyEmpty
+	}
+	bytes, err := db.Get(key)
+	if err != nil {
+		return false, err
+	}
+	return bytes != nil, nil
+}
+
+// Set implements DB.
+func (db *PebbleDB) Set(key []byte, value []byte) error {
+	if len(key) == 0 {
+		return errKeyEmpty
+	}
+	if value == nil {
+		return errValueNil
+	}
+	err := db.db.Set(key, value, pebble.NoSync)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// SetSync implements DB.
+func (db *PebbleDB) SetSync(key []byte, value []byte) error {
+	if len(key) == 0 {
+		return errKeyEmpty
+	}
+	if value == nil {
+		return errValueNil
+	}
+	err := db.db.Set(key, value, pebble.Sync)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Delete implements DB.
+func (db *PebbleDB) Delete(key []byte) error {
+	if len(key) == 0 {
+		return errKeyEmpty
+	}
+	err := db.db.Delete(key, pebble.NoSync)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// DeleteSync implements DB.
+func (db PebbleDB) DeleteSync(key []byte) error {
+	if len(key) == 0 {
+		return errKeyEmpty
+	}
+	err := db.db.Delete(key, pebble.Sync)
+	if err != nil {
+		return nil
+	}
+	return nil
+}
+
+func (db *PebbleDB) DB() *pebble.DB {
+	return db.db
+}
+
+// Close implements DB.
+func (db PebbleDB) Close() error {
+	db.db.Close()
+	return nil
+}
+
+// Print implements DB.
+func (db *PebbleDB) Print() error {
+	itr, err := db.Iterator(nil, nil)
+	if err != nil {
+		return err
+	}
+	defer itr.Close()
+	for ; itr.Valid(); itr.Next() {
+		key := itr.Key()
+		value := itr.Value()
+		fmt.Printf("[%X]:\t[%X]\n", key, value)
+	}
+	return nil
+}
+
+// Stats implements DB.
+func (db *PebbleDB) Stats() map[string]string {
+	/*
+		keys := []string{"rocksdb.stats"}
+		stats := make(map[string]string, len(keys))
+		for _, key := range keys {
+			stats[key] = db.(key)
+		}
+	*/
+	stats := make(map[string]string, 1)
+        stats["Metrics"] = fmt.Sprint(db.db.Metrics())
+        return stats
+}
+
+// NewBatch implements DB.
+func (db *PebbleDB) NewBatch() Batch {
+	return newPebbleDBBatch(db)
+}
+
+// Iterator implements DB.
+func (db *PebbleDB) Iterator(start, end []byte) (Iterator, error) {
+	if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) {
+		return nil, errKeyEmpty
+	}
+	o := pebble.IterOptions{
+		LowerBound: start,
+		UpperBound: end,
+	}
+	itr := db.db.NewIter(&o)
+	itr.First()
+
+	return newPebbleDBIterator(itr, start, end, false), nil
+}
+
+// ReverseIterator implements DB.
+func (db *PebbleDB) ReverseIterator(start, end []byte) (Iterator, error) {
+	if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) {
+		return nil, errKeyEmpty
+	}
+	o := pebble.IterOptions{
+		LowerBound: start,
+		UpperBound: end,
+	}
+	itr := db.db.NewIter(&o)
+	itr.Last()
+	return newPebbleDBIterator(itr, start, end, true), nil
+}

pebble_batch.go

@@ -0,0 +1,86 @@
+//go:build pebbledb
+
+package db
+
+import "github.com/cockroachdb/pebble"
+
+type pebbleDBBatch struct {
+	db    *PebbleDB
+	batch *pebble.Batch
+}
+
+var _ Batch = (*pebbleDBBatch)(nil)
+
+func newPebbleDBBatch(db *PebbleDB) *pebbleDBBatch {
+	return &pebbleDBBatch{
+		db:    *PebbleDB,
+		batch: db.db.NewBatch(),
+	}
+}
+
+// Set implements Batch.
+func (b *pebbleDBBatch) Set(key, value []byte) error {
+	if len(key) == 0 {
+		return errKeyEmpty
+	}
+	if value == nil {
+		return errValueNil
+	}
+	if b.batch == nil {
+		return errBatchClosed
+	}
+	b.batch.Set(key, value, nil)
+	return nil
+}
+
+// Delete implements Batch.
+func (b *pebbleDBBatch) Delete(key []byte) error {
+	if len(key) == 0 {
+		return errKeyEmpty
+	}
+	if b.batch == nil {
+		return errBatchClosed
+	}
+	b.batch.Delete(key, nil)
+	return nil
+}
+
+// Write implements Batch.
+func (b *pebbleDBBatch) Write() error {
+	if b.batch == nil {
+		return errBatchClosed
+	}
+	err := b.batch.Commit(pebble.NoSync)
+	if err != nil {
+		return err
+	}
+	// Make sure batch cannot be used afterwards. Callers should still call Close(), for errors.
+
+	return b.Close()
+}
+
+// WriteSync implements Batch.
+func (b *pebbleDBBatch) WriteSync() error {
+	if b.batch == nil {
+		return errBatchClosed
+	}
+	err := b.batch.Commit(pebble.Sync)
+	if err != nil {
+		return err
+	}
+	// Make sure batch cannot be used afterwards. Callers should still call Close(), for errors.
+	return b.Close()
+}
+
+// Close implements Batch.
+func (b *pebbleDBBatch) Close() error {
+	if b.batch != nil {
+		err := b.batch.Close()
+		if err != nil {
+			return err
+		}
+		b.batch = nil
+	}
+
+	return nil
+}