cmd, core, ethdb, node: move chain freezer one folder deeper (#25487)

* cmd, core, ethdb, node: create chain freezer in a sub folder

* core/rawdb: remove unused code

* core, ethdb, node: add AncientDatadir API back

* cmd, core: extend freezer info dump for sub-ancient-store

* core/rawdb: rework freezer inspector

* core/rawdb: address comments from Peter

* core/rawdb: fix build issue

Author: rjl493456442

cmd/geth/dbcmd.go

@@ -22,7 +22,6 @@ import (
 	"os"
 	"os/signal"
 	"path/filepath"
-	"sort"
 	"strconv"
 	"strings"
 	"syscall"
@@ -160,8 +159,8 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 	dbDumpFreezerIndex = &cli.Command{
 		Action:    freezerInspect,
 		Name:      "freezer-index",
-		Usage:     "Dump out the index of a given freezer type",
-		ArgsUsage: "<type> <start (int)> <end (int)>",
+		Usage:     "Dump out the index of a specific freezer table",
+		ArgsUsage: "<freezer-type> <table-type> <start (int)> <end (int)>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
 		}, utils.NetworkFlags, utils.DatabasePathFlags),
@@ -275,7 +274,7 @@ func inspect(ctx *cli.Context) error {
 		start  []byte
 	)
 	if ctx.NArg() > 2 {
-		return fmt.Errorf("Max 2 arguments: %v", ctx.Command.ArgsUsage)
+		return fmt.Errorf("max 2 arguments: %v", ctx.Command.ArgsUsage)
 	}
 	if ctx.NArg() >= 1 {
 		if d, err := hexutil.Decode(ctx.Args().Get(0)); err != nil {
@@ -536,43 +535,35 @@ func dbDumpTrie(ctx *cli.Context) error {
 }
 
 func freezerInspect(ctx *cli.Context) error {
-	var (
-		start, end    int64
-		disableSnappy bool
-		err           error
-	)
-	if ctx.NArg() < 3 {
+	if ctx.NArg() < 4 {
 		return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
 	}
-	kind := ctx.Args().Get(0)
-	if noSnap, ok := rawdb.FreezerNoSnappy[kind]; !ok {
-		var options []string
-		for opt := range rawdb.FreezerNoSnappy {
-			options = append(options, opt)
-		}
-		sort.Strings(options)
-		return fmt.Errorf("Could read freezer-type '%v'. Available options: %v", kind, options)
-	} else {
-		disableSnappy = noSnap
-	}
-	if start, err = strconv.ParseInt(ctx.Args().Get(1), 10, 64); err != nil {
-		log.Info("Could read start-param", "error", err)
+	var (
+		freezer = ctx.Args().Get(0)
+		table   = ctx.Args().Get(1)
+	)
+	start, err := strconv.ParseInt(ctx.Args().Get(2), 10, 64)
+	if err != nil {
+		log.Info("Could not read start-param", "err", err)
 		return err
 	}
-	if end, err = strconv.ParseInt(ctx.Args().Get(2), 10, 64); err != nil {
-		log.Info("Could read count param", "error", err)
+	end, err := strconv.ParseInt(ctx.Args().Get(3), 10, 64)
+	if err != nil {
+		log.Info("Could not read count param", "err", err)
 		return err
 	}
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
-	path := filepath.Join(stack.ResolvePath("chaindata"), "ancient")
-	log.Info("Opening freezer", "location", path, "name", kind)
-	if f, err := rawdb.NewFreezerTable(path, kind, disableSnappy, true); err != nil {
+
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+
+	ancient, err := db.AncientDatadir()
+	if err != nil {
+		log.Info("Failed to retrieve ancient root", "err", err)
 		return err
-	} else {
-		f.DumpIndex(start, end)
 	}
-	return nil
+	return rawdb.InspectFreezerTable(ancient, freezer, table, start, end)
 }
 
 func importLDBdata(ctx *cli.Context) error {

cmd/utils/flags.go

@@ -91,7 +91,7 @@ var (
 	}
 	AncientFlag = &flags.DirectoryFlag{
 		Name:     "datadir.ancient",
-		Usage:    "Data directory for ancient chain segments (default = inside chaindata)",
+		Usage:    "Root directory for ancient data (default = inside chaindata)",
 		Category: flags.EthCategory,
 	}
 	MinFreeDiskSpaceFlag = &flags.DirectoryFlag{

core/rawdb/accessors_chain.go

@@ -37,7 +37,7 @@ import (
 func ReadCanonicalHash(db ethdb.Reader, number uint64) common.Hash {
 	var data []byte
 	db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
-		data, _ = reader.Ancient(freezerHashTable, number)
+		data, _ = reader.Ancient(chainFreezerHashTable, number)
 		if len(data) == 0 {
 			// Get it by hash from leveldb
 			data, _ = db.Get(headerHashKey(number))
@@ -335,7 +335,7 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu
 	}
 	// read remaining from ancients
 	max := count * 700
-	data, err := db.AncientRange(freezerHeaderTable, i+1-count, count, max)
+	data, err := db.AncientRange(chainFreezerHeaderTable, i+1-count, count, max)
 	if err == nil && uint64(len(data)) == count {
 		// the data is on the order [h, h+1, .., n] -- reordering needed
 		for i := range data {
@@ -352,7 +352,7 @@ func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValu
 		// First try to look up the data in ancient database. Extra hash
 		// comparison is necessary since ancient database only maintains
 		// the canonical data.
-		data, _ = reader.Ancient(freezerHeaderTable, number)
+		data, _ = reader.Ancient(chainFreezerHeaderTable, number)
 		if len(data) > 0 && crypto.Keccak256Hash(data) == hash {
 			return nil
 		}
@@ -428,7 +428,7 @@ func deleteHeaderWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number
 // isCanon is an internal utility method, to check whether the given number/hash
 // is part of the ancient (canon) set.
 func isCanon(reader ethdb.AncientReaderOp, number uint64, hash common.Hash) bool {
-	h, err := reader.Ancient(freezerHashTable, number)
+	h, err := reader.Ancient(chainFreezerHashTable, number)
 	if err != nil {
 		return false
 	}
@@ -444,7 +444,7 @@ func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue
 	db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
 		// Check if the data is in ancients
 		if isCanon(reader, number, hash) {
-			data, _ = reader.Ancient(freezerBodiesTable, number)
+			data, _ = reader.Ancient(chainFreezerBodiesTable, number)
 			return nil
 		}
 		// If not, try reading from leveldb
@@ -459,7 +459,7 @@ func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue
 func ReadCanonicalBodyRLP(db ethdb.Reader, number uint64) rlp.RawValue {
 	var data []byte
 	db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
-		data, _ = reader.Ancient(freezerBodiesTable, number)
+		data, _ = reader.Ancient(chainFreezerBodiesTable, number)
 		if len(data) > 0 {
 			return nil
 		}
@@ -527,7 +527,7 @@ func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
 	db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
 		// Check if the data is in ancients
 		if isCanon(reader, number, hash) {
-			data, _ = reader.Ancient(freezerDifficultyTable, number)
+			data, _ = reader.Ancient(chainFreezerDifficultyTable, number)
 			return nil
 		}
 		// If not, try reading from leveldb
@@ -587,7 +587,7 @@ func ReadReceiptsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawVa
 	db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
 		// Check if the data is in ancients
 		if isCanon(reader, number, hash) {
-			data, _ = reader.Ancient(freezerReceiptTable, number)
+			data, _ = reader.Ancient(chainFreezerReceiptTable, number)
 			return nil
 		}
 		// If not, try reading from leveldb
@@ -819,19 +819,19 @@ func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts
 
 func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts []*types.ReceiptForStorage, td *big.Int) error {
 	num := block.NumberU64()
-	if err := op.AppendRaw(freezerHashTable, num, block.Hash().Bytes()); err != nil {
+	if err := op.AppendRaw(chainFreezerHashTable, num, block.Hash().Bytes()); err != nil {
 		return fmt.Errorf("can't add block %d hash: %v", num, err)
 	}
-	if err := op.Append(freezerHeaderTable, num, header); err != nil {
+	if err := op.Append(chainFreezerHeaderTable, num, header); err != nil {
 		return fmt.Errorf("can't append block header %d: %v", num, err)
 	}
-	if err := op.Append(freezerBodiesTable, num, block.Body()); err != nil {
+	if err := op.Append(chainFreezerBodiesTable, num, block.Body()); err != nil {
 		return fmt.Errorf("can't append block body %d: %v", num, err)
 	}
-	if err := op.Append(freezerReceiptTable, num, receipts); err != nil {
+	if err := op.Append(chainFreezerReceiptTable, num, receipts); err != nil {
 		return fmt.Errorf("can't append block %d receipts: %v", num, err)
 	}
-	if err := op.Append(freezerDifficultyTable, num, td); err != nil {
+	if err := op.Append(chainFreezerDifficultyTable, num, td); err != nil {
 		return fmt.Errorf("can't append block %d total difficulty: %v", num, err)
 	}
 	return nil

core/rawdb/ancient_scheme.go

@@ -0,0 +1,86 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rawdb
+
+import "fmt"
+
+// The list of table names of chain freezer.
+const (
+	// chainFreezerHeaderTable indicates the name of the freezer header table.
+	chainFreezerHeaderTable = "headers"
+
+	// chainFreezerHashTable indicates the name of the freezer canonical hash table.
+	chainFreezerHashTable = "hashes"
+
+	// chainFreezerBodiesTable indicates the name of the freezer block body table.
+	chainFreezerBodiesTable = "bodies"
+
+	// chainFreezerReceiptTable indicates the name of the freezer receipts table.
+	chainFreezerReceiptTable = "receipts"
+
+	// chainFreezerDifficultyTable indicates the name of the freezer total difficulty table.
+	chainFreezerDifficultyTable = "diffs"
+)
+
+// chainFreezerNoSnappy configures whether compression is disabled for the ancient-tables.
+// Hashes and difficulties don't compress well.
+var chainFreezerNoSnappy = map[string]bool{
+	chainFreezerHeaderTable:     false,
+	chainFreezerHashTable:       true,
+	chainFreezerBodiesTable:     false,
+	chainFreezerReceiptTable:    false,
+	chainFreezerDifficultyTable: true,
+}
+
+// The list of identifiers of ancient stores.
+var (
+	chainFreezerName = "chain" // the folder name of chain segment ancient store.
+)
+
+// freezers the collections of all builtin freezers.
+var freezers = []string{chainFreezerName}
+
+// InspectFreezerTable dumps out the index of a specific freezer table. The passed
+// ancient indicates the path of root ancient directory where the chain freezer can
+// be opened. Start and end specify the range for dumping out indexes.
+// Note this function can only be used for debugging purposes.
+func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
+	var (
+		path   string
+		tables map[string]bool
+	)
+	switch freezerName {
+	case chainFreezerName:
+		path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
+	default:
+		return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
+	}
+	noSnappy, exist := tables[tableName]
+	if !exist {
+		var names []string
+		for name := range tables {
+			names = append(names, name)
+		}
+		return fmt.Errorf("unknown table, supported ones: %v", names)
+	}
+	table, err := newFreezerTable(path, tableName, noSnappy, true)
+	if err != nil {
+		return err
+	}
+	table.dumpIndexStdout(start, end)
+	return nil
+}

core/rawdb/chain_freezer.go

@@ -278,19 +278,19 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash
 			}
 
 			// Write to the batch.
-			if err := op.AppendRaw(freezerHashTable, number, hash[:]); err != nil {
+			if err := op.AppendRaw(chainFreezerHashTable, number, hash[:]); err != nil {
 				return fmt.Errorf("can't write hash to Freezer: %v", err)
 			}
-			if err := op.AppendRaw(freezerHeaderTable, number, header); err != nil {
+			if err := op.AppendRaw(chainFreezerHeaderTable, number, header); err != nil {
 				return fmt.Errorf("can't write header to Freezer: %v", err)
 			}
-			if err := op.AppendRaw(freezerBodiesTable, number, body); err != nil {
+			if err := op.AppendRaw(chainFreezerBodiesTable, number, body); err != nil {
 				return fmt.Errorf("can't write body to Freezer: %v", err)
 			}
-			if err := op.AppendRaw(freezerReceiptTable, number, receipts); err != nil {
+			if err := op.AppendRaw(chainFreezerReceiptTable, number, receipts); err != nil {
 				return fmt.Errorf("can't write receipts to Freezer: %v", err)
 			}
-			if err := op.AppendRaw(freezerDifficultyTable, number, td); err != nil {
+			if err := op.AppendRaw(chainFreezerDifficultyTable, number, td); err != nil {
 				return fmt.Errorf("can't write td to Freezer: %v", err)
 			}
 

core/rawdb/chain_iterator.go

@@ -50,7 +50,7 @@ func InitDatabaseFromFreezer(db ethdb.Database) {
 		if i+count > frozen {
 			count = frozen - i
 		}
-		data, err := db.AncientRange(freezerHashTable, i, count, 32*count)
+		data, err := db.AncientRange(chainFreezerHashTable, i, count, 32*count)
 		if err != nil {
 			log.Crit("Failed to init database from freezer", "err", err)
 		}