fix(pebble.go): add intermediate commit to batch if nearing hard limit

pebble.go

@@ -77,6 +77,8 @@ type PebbleDB struct {
 
 var _ DB = (*PebbleDB)(nil)
 
+const flushThreshold = 3_500_000_000 // ~3.5 GB
+
 func NewPebbleDB(name, dir string, opts Options) (DB, error) {
 	do := &pebble.Options{
 		Logger: &fatalLogger{}, // pebble info logs are messing up the logs
@@ -306,6 +308,15 @@ func (b *pebbleDBBatch) Set(key, value []byte) error {
 	if b.batch == nil {
 		return errBatchClosed
 	}
+
+	// Prevent Pebble batch from exceeding 4 GB hard limit
+	if b.batch.Len() > flushThreshold {
+		if err := b.batch.Commit(pebble.Sync); err != nil {
+			return err
+		}
+		b.batch.Reset()
+	}
+
 	return b.batch.Set(key, value, nil)
 }
 
@@ -317,6 +328,15 @@ func (b *pebbleDBBatch) Delete(key []byte) error {
 	if b.batch == nil {
 		return errBatchClosed
 	}
+
+	// Prevent Pebble batch from exceeding 4 GB hard limit
+	if b.batch.Len() > flushThreshold {
+		if err := b.batch.Commit(pebble.Sync); err != nil {
+			return err
+		}
+		b.batch.Reset()
+	}
+
 	return b.batch.Delete(key, nil)
 }