Mdbx - make it default db. Lazy buckets renaming. (erigontech#1982)

cmd/cons/commands/root.go

@@ -52,7 +52,7 @@ func must(err error) {
 func withDatadir(cmd *cobra.Command) {
 	cmd.Flags().StringVar(&datadir, "datadir", paths.DefaultDataDir(), "directory where databases and temporary files are kept")
 	must(cmd.MarkFlagDirname("datadir"))
-	cmd.Flags().StringVar(&database, "database", "", "lmdb|mdbx")
+	cmd.Flags().StringVar(&database, "database", "mdbx", "lmdb|mdbx")
 }
 
 func withApiAddr(cmd *cobra.Command) {
@@ -69,15 +69,15 @@ func openDatabase(path string) *ethdb.ObjectDatabase {
 }
 
 func openKV(path string, exclusive bool) ethdb.RwKV {
-	if database == "mdbx" {
-		opts := ethdb.NewMDBX().Path(path)
+	if database == "lmdb" {
+		opts := ethdb.NewLMDB().Path(path)
 		if exclusive {
 			opts = opts.Exclusive()
 		}
 		return opts.MustOpen()
 	}
 
-	opts := ethdb.NewLMDB().Path(path)
+	opts := ethdb.NewMDBX().Path(path)
 	if exclusive {
 		opts = opts.Exclusive()
 	}

cmd/hack/hack.go

@@ -1,5 +1,3 @@
-//+build mdbx
-
 package main
 
 import (

cmd/integration/commands/flags.go

@@ -91,7 +91,7 @@ func withDatadir2(cmd *cobra.Command) {
 	cmd.Flags().String(utils.DataDirFlag.Name, paths.DefaultDataDir(), utils.DataDirFlag.Usage)
 	must(cmd.MarkFlagDirname(utils.DataDirFlag.Name))
 	must(cmd.MarkFlagRequired(utils.DataDirFlag.Name))
-	cmd.Flags().StringVar(&database, "database", "lmdb", "lmdb|mdbx")
+	cmd.Flags().StringVar(&database, "database", "mdbx", "lmdb|mdbx")
 	cmd.Flags().IntVar(&databaseVerbosity, "database.verbosity", 2, "Enabling internal db logs. Very high verbosity levels may require recompile db. Default: 2, means warning.")
 }
 
@@ -107,7 +107,7 @@ func withDatadir(cmd *cobra.Command) {
 	cmd.Flags().StringVar(&snapshotDir, "snapshot.dir", "", "snapshot dir")
 	must(cmd.MarkFlagDirname("snapshot.dir"))
 
-	cmd.Flags().StringVar(&database, "database", "lmdb", "lmdb|mdbx")
+	cmd.Flags().StringVar(&database, "database", "mdbx", "lmdb|mdbx")
 	cmd.Flags().IntVar(&databaseVerbosity, "database.verbosity", 2, "Enabling internal db logs. Very high verbosity levels may require recompile db. Default: 2, means warning")
 }
 

cmd/integration/commands/root.go

@@ -69,8 +69,8 @@ func openDatabase(path string, applyMigrations bool) *ethdb.ObjectDatabase {
 }
 
 func openKV(path string, exclusive bool) ethdb.RwKV {
-	if database == "mdbx" {
-		opts := ethdb.NewMDBX().Path(path)
+	if database == "lmdb" {
+		opts := ethdb.NewLMDB().Path(path)
 		if exclusive {
 			opts = opts.Exclusive()
 		}
@@ -82,10 +82,12 @@ func openKV(path string, exclusive bool) ethdb.RwKV {
 		if databaseVerbosity != -1 {
 			opts = opts.DBVerbosity(ethdb.DBVerbosityLvl(databaseVerbosity))
 		}
-		return opts.MustOpen()
+		kv := opts.MustOpen()
+		metrics.AddCallback(kv.CollectMetrics)
+		return kv
 	}
 
-	opts := ethdb.NewLMDB().Path(path)
+	opts := ethdb.NewMDBX().Path(path)
 	if exclusive {
 		opts = opts.Exclusive()
 	}

cmd/rpcdaemon/cli/config.go

@@ -57,7 +57,7 @@ func RootCommand() (*cobra.Command, *Flags) {
 	cfg := &Flags{}
 	rootCmd.PersistentFlags().StringVar(&cfg.PrivateApiAddr, "private.api.addr", "127.0.0.1:9090", "private api network address, for example: 127.0.0.1:9090, empty string means not to start the listener. do not expose to public network. serves remote database interface")
 	rootCmd.PersistentFlags().StringVar(&cfg.Datadir, "datadir", "", "path to turbo-geth working directory")
-	rootCmd.PersistentFlags().StringVar(&cfg.Database, "database", "lmdb", "lmdb|mdbx engines")
+	rootCmd.PersistentFlags().StringVar(&cfg.Database, "database", "mdbx", "lmdb|mdbx engines")
 	rootCmd.PersistentFlags().StringVar(&cfg.Chaindata, "chaindata", "", "path to the database")
 	rootCmd.PersistentFlags().StringVar(&cfg.SnapshotDir, "snapshot.dir", "", "path to snapshot dir(only for chaindata mode)")
 	rootCmd.PersistentFlags().StringVar(&cfg.SnapshotMode, "snapshot.mode", "", `Configures the storage mode of the app(only for chaindata mode):

cmd/snapshots/generator/commands/root.go

@@ -91,7 +91,7 @@ func withDatadir(cmd *cobra.Command) {
 	cmd.Flags().StringVar(&snapshotDir, "snapshot.dir", "", "snapshot dir")
 	must(cmd.MarkFlagDirname("snapshot.dir"))
 
-	cmd.Flags().StringVar(&database, "database", "", "lmdb|mdbx")
+	cmd.Flags().StringVar(&database, "database", "mdbx", "lmdb|mdbx")
 }
 
 func withSnapshotFile(cmd *cobra.Command) {

cmd/state/commands/global_flags_vars.go

@@ -39,7 +39,7 @@ func withDatadir(cmd *cobra.Command) {
 	cmd.Flags().StringVar(&snapshotDir, "snapshot.dir", "", "snapshot dir")
 	must(cmd.MarkFlagDirname("snapshot.dir"))
 
-	cmd.Flags().StringVar(&database, "database", "", "lmdb|mdbx")
+	cmd.Flags().StringVar(&database, "database", "mdbx", "lmdb|mdbx")
 }
 
 func withStatsfile(cmd *cobra.Command) {

common/dbutils/bucket.go

@@ -69,7 +69,6 @@ const (
 	HashedAccountsBucket   = "hashed_accounts"
 	HashedStorageBucket    = "hashed_storage"
 	CurrentStateBucketOld2 = "CST2"
-	CurrentStateBucketOld1 = "CST"
 
 	//key - address + shard_id_u64
 	//value - roaring bitmap  - list of block where it changed
@@ -135,7 +134,6 @@ Invariants:
 */
 const TrieOfAccountsBucket = "trie_account"
 const TrieOfStorageBucket = "trie_storage"
-const IntermediateTrieHashBucketOld1 = "iTh"
 const IntermediateTrieHashBucketOld2 = "iTh2"
 
 const (
@@ -218,10 +216,64 @@ const (
 	// in case of bug-report developer can ask content of this bucket
 	Migrations = "migrations"
 
-	Sequence = "sequence" // tbl_name -> seq_u64
-
+	Sequence      = "sequence" // tbl_name -> seq_u64
+	HeadHeaderKey = "LastHeader"
 )
 
+var Rename = map[string]string{
+	PlainStateBucket:          "PlainState",
+	PlainContractCodeBucket:   "PlainCodeHash",
+	AccountChangeSetBucket:    "AccountChangeSet",
+	StorageChangeSetBucket:    "StorageChangeSet",
+	HashedAccountsBucket:      "HashedAccount",
+	HashedStorageBucket:       "HashedStorage",
+	AccountsHistoryBucket:     "AccountHistory",
+	StorageHistoryBucket:      "StorageHistory",
+	CodeBucket:                "Code",
+	ContractCodeBucket:        "HashedCodeHash",
+	IncarnationMapBucket:      "IncarnationMap",
+	TrieOfAccountsBucket:      "TrieAccount",
+	TrieOfStorageBucket:       "TrieStorage",
+	DatabaseInfoBucket:        "DbInfo",
+	SnapshotInfoBucket:        "SnapshotInfo",
+	BittorrentInfoBucket:      "BittorrentInfo",
+	HeadersSnapshotInfoBucket: "HeadersSnapshotInfo",
+	BodiesSnapshotInfoBucket:  "BodiesSnapshotInfo",
+	StateSnapshotInfoBucket:   "StateSnapshotInfo",
+	HeaderNumberBucket:        "HeaderNumber",
+	HeaderCanonicalBucket:     "CanonicalHeader",
+	HeadersBucket:             "Header",
+	HeaderTDBucket:            "HeadersTotalDifficulty",
+	BlockBodyPrefix:           "BlockBody",
+	EthTx:                     "BlockTransaction",
+	BlockReceiptsPrefix:       "Receipt",
+	Log:                       "TransactionLog",
+	LogTopicIndex:             "LogTopicIndex",
+	LogAddressIndex:           "LogAddressIndex",
+	CallTraceSet:              "CallTraceSet",
+	CallFromIndex:             "CallFromIndex",
+	CallToIndex:               "CallToIndex",
+	TxLookupPrefix:            "BlockTransactionLookup",
+	BloomBitsPrefix:           "BloomBits",
+	PreimagePrefix:            "Preimage",
+	ConfigPrefix:              "Config",
+	BloomBitsIndexPrefix:      "BloomBitsIndex",
+	SyncStageProgress:         "SyncStage",
+	SyncStageUnwind:           "SyncStageUnwind",
+	CliqueBucket:              "Clique",
+	CliqueSeparateBucket:      "CliqueSeparate",
+	CliqueSnapshotBucket:      "CliqueSnapshot",
+	CliqueLastSnapshotBucket:  "CliqueLastSnapshot",
+	InodesBucket:              "Inode",
+	Senders:                   "TxSender",
+	HeadBlockKey:              "LastBlock",
+	InvalidBlock:              "InvalidBlock",
+	UncleanShutdown:           "UncleanShutdown",
+	Migrations:                "Migration",
+	Sequence:                  "Sequence",
+	HeadHeaderKey:             "LastHeader",
+}
+
 // Keys
 var (
 	//StorageModePruning - does node prune.
@@ -237,8 +289,6 @@ var (
 
 	DBSchemaVersionKey = []byte("dbVersion")
 
-	HeadHeaderKey = "LastHeader"
-
 	SnapshotHeadersHeadNumber = "SnapshotLastHeaderNumber"
 	SnapshotHeadersHeadHash   = "SnapshotLastHeaderHash"
 	SnapshotBodyHeadNumber    = "SnapshotLastBodyNumber"
@@ -253,7 +303,6 @@ var (
 // This list will be sorted in `init` method.
 // BucketsConfigs - can be used to find index in sorted version of Buckets list by name
 var Buckets = []string{
-	CurrentStateBucketOld2,
 	AccountsHistoryBucket,
 	StorageHistoryBucket,
 	CodeBucket,
@@ -297,7 +346,6 @@ var Buckets = []string{
 	TrieOfStorageBucket,
 	HashedAccountsBucket,
 	HashedStorageBucket,
-	IntermediateTrieHashBucketOld2,
 	BittorrentInfoBucket,
 	HeaderCanonicalBucket,
 	HeadersBucket,
@@ -306,11 +354,11 @@ var Buckets = []string{
 
 // DeprecatedBuckets - list of buckets which can be programmatically deleted - for example after migration
 var DeprecatedBuckets = []string{
+	IntermediateTrieHashBucketOld2,
+	CurrentStateBucketOld2,
 	SyncStageProgressOld1,
 	SyncStageUnwindOld1,
-	CurrentStateBucketOld1,
 	PlainStateBucketOld1,
-	IntermediateTrieHashBucketOld1,
 	HeaderPrefixOld,
 	CliqueBucket,
 }

common/dbutils/rename.go

@@ -0,0 +1 @@
+package dbutils

ethdb/kv_mdbx.go

@@ -1,5 +1,3 @@
-//+build mdbx
-
 package ethdb
 
 import (
@@ -539,7 +537,6 @@ func (db *MdbxKV) Update(ctx context.Context, f func(tx RwTx) error) (err error)
 
 func (tx *MdbxTx) CreateBucket(name string) error {
 	cnfCopy := tx.db.buckets[name]
-
 	var dcmp mdbx.CmpFunc
 	switch cnfCopy.CustomDupComparator {
 	case dbutils.DupCmpSuffix32:
@@ -563,6 +560,26 @@ func (tx *MdbxTx) CreateBucket(name string) error {
 		return nil
 	}
 
+	// if bucket with this name not found - check renamed one
+	rename := dbutils.Rename[name]
+
+	dbi, err = tx.tx.OpenDBI(rename, mdbx.DBAccede, nil, dcmp)
+	if err != nil && !mdbx.IsNotFound(err) {
+		return fmt.Errorf("create bucket: %s, %w", name, err)
+	}
+	if err == nil {
+		cnfCopy.DBI = dbutils.DBI(dbi)
+		var flags uint
+		flags, err = tx.tx.Flags(dbi)
+		if err != nil {
+			return err
+		}
+		cnfCopy.Flags = dbutils.BucketFlags(flags)
+
+		tx.db.buckets[name] = cnfCopy
+		return nil
+	}
+
 	// if bucket doesn't exists - create it
 
 	var flags = tx.db.buckets[name].Flags
@@ -579,7 +596,11 @@ func (tx *MdbxTx) CreateBucket(name string) error {
 		return fmt.Errorf("some not supported flag provided for bucket")
 	}
 
-	dbi, err = tx.tx.OpenDBI(name, nativeFlags, nil, dcmp)
+	if rename != "" {
+		dbi, err = tx.tx.OpenDBI(rename, nativeFlags, nil, dcmp)
+	} else {
+		dbi, err = tx.tx.OpenDBI(name, nativeFlags, nil, dcmp)
+	}
 	if err != nil {
 		return fmt.Errorf("create bucket: %s, %w", name, err)
 	}
@@ -1350,7 +1371,6 @@ func (c *MdbxCursor) Append(k []byte, v []byte) error {
 		}
 		return nil
 	}
-
 	if err := c.append(k, v); err != nil {
 		return fmt.Errorf("bucket: %s, %w", c.bucketName, err)
 	}

ethdb/mdbx/mdbx.go

@@ -1,5 +1,3 @@
-// +build mdbx
-
 /*
 Package lmdb provides bindings to the lmdb C API.  The package bindings are
 fairly low level and are designed to provide a minimal interface that prevents

ethdb/memory_database.go

@@ -41,7 +41,7 @@ func NewMemKV() RwKV {
 		// mdbx is too slow for our tests currently, so we keep
 		// lmdb as our in-mem db
 		// with mdbx tests time out, especially ./tests package
-		return NewLMDB().InMem().MustOpen()
+		return NewMDBX().InMem().MustOpen()
 	}
 }