From 378199180766f5a66fe57fd07b55d8a2cd57b6a5 Mon Sep 17 00:00:00 2001 From: JukLee0ira Date: Thu, 19 Sep 2024 14:40:35 +0800 Subject: [PATCH] core: remove unnecessary fields in log (17016) --- core/blockchain.go | 8 ++- core/rawdb/accessors_chain.go | 7 +++ core/rawdb/accessors_indexes.go | 84 ++++++++++++++++++++++++++ core/rawdb/accessors_indexes_test.go | 90 ++++++++++++++++++++++++++++ core/rawdb/accessors_metadata.go | 37 ++++++++++++ core/rawdb/schema.go | 20 +++++++ core/types/log.go | 41 +++++++------ core/types/receipt.go | 21 +++++++ eth/backend.go | 3 +- les/handler.go | 4 +- les/handler_test.go | 4 +- les/protocol.go | 3 +- 12 files changed, 297 insertions(+), 25 deletions(-) create mode 100644 core/rawdb/accessors_indexes.go create mode 100644 core/rawdb/accessors_indexes_test.go create mode 100644 core/rawdb/accessors_metadata.go diff --git a/core/blockchain.go b/core/blockchain.go index 006e94fd0e05..6926d2286638 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -75,7 +75,13 @@ const ( triesInMemory = 128 // BlockChainVersion ensures that an incompatible database forces a resync from scratch. - BlockChainVersion = 3 + // + // During the process of upgrading the database version from 3 to 4, + // the following incompatible database changes were added. + // * the `BlockNumber`, `TxHash`, `TxIndex`, `BlockHash` and `Index` fields of log are deleted + // * the `Bloom` field of receipt is deleted + // * the `BlockIndex` and `TxIndex` fields of txlookup are deleted + BlockChainVersion = 4 // Maximum length of chain to cache by block's number blocksHashCacheLimit = 900 diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 6790fe141c58..1d224c0d60f1 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -239,6 +239,13 @@ func WriteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64, rec } } +// DeleteReceipts removes all receipt data associated with a block hash. +func DeleteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { + if err := db.Delete(blockReceiptsKey(number, hash)); err != nil { + log.Crit("Failed to delete block receipts", "err", err) + } +} + // storedReceiptRLP is the storage encoding of a receipt. // Re-definition in core/types/receipt.go. type storedReceiptRLP struct { diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go new file mode 100644 index 000000000000..aa970bd2fa43 --- /dev/null +++ b/core/rawdb/accessors_indexes.go @@ -0,0 +1,84 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "github.com/XinFinOrg/XDPoSChain/common" + "github.com/XinFinOrg/XDPoSChain/core/types" + "github.com/XinFinOrg/XDPoSChain/ethdb" + "github.com/XinFinOrg/XDPoSChain/log" + "github.com/XinFinOrg/XDPoSChain/rlp" +) + +// ReadTxLookupEntry retrieves the positional metadata associated with a transaction +// hash to allow retrieving the transaction or receipt by hash. +func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) common.Hash { + data, _ := db.Get(txLookupKey(hash)) + if len(data) == 0 { + return common.Hash{} + } + if len(data) == common.HashLength { + return common.BytesToHash(data) + } + // Probably it's legacy txlookup entry data, try to decode it. + var entry LegacyTxLookupEntry + if err := rlp.DecodeBytes(data, &entry); err != nil { + log.Error("Invalid transaction lookup entry RLP", "hash", hash, "blob", data, "err", err) + return common.Hash{} + } + return entry.BlockHash +} + +// WriteTxLookupEntries stores a positional metadata for every transaction from +// a block, enabling hash based transaction and receipt lookups. +func WriteTxLookupEntries(db ethdb.KeyValueWriter, block *types.Block) { + for _, tx := range block.Transactions() { + if err := db.Put(txLookupKey(tx.Hash()), block.Hash().Bytes()); err != nil { + log.Crit("Failed to store transaction lookup entry", "err", err) + } + } +} + +// DeleteTxLookupEntry removes all transaction data associated with a hash. +func DeleteTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash) { + db.Delete(txLookupKey(hash)) +} + +// ReadTransaction retrieves a specific transaction from the database, along with +// its added positional metadata. +func ReadTransaction(db ethdb.Reader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { + blockHash := ReadTxLookupEntry(db, hash) + if blockHash == (common.Hash{}) { + return nil, common.Hash{}, 0, 0 + } + blockNumber := ReadHeaderNumber(db, blockHash) + if blockNumber == nil { + return nil, common.Hash{}, 0, 0 + } + body := ReadBody(db, blockHash, *blockNumber) + if body == nil { + log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash) + return nil, common.Hash{}, 0, 0 + } + for txIndex, tx := range body.Transactions { + if tx.Hash() == hash { + return tx, blockHash, *blockNumber, uint64(txIndex) + } + } + log.Error("Transaction not found", "number", blockNumber, "hash", blockHash, "txhash", hash) + return nil, common.Hash{}, 0, 0 +} diff --git a/core/rawdb/accessors_indexes_test.go b/core/rawdb/accessors_indexes_test.go new file mode 100644 index 000000000000..c7a18633fad9 --- /dev/null +++ b/core/rawdb/accessors_indexes_test.go @@ -0,0 +1,90 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "math/big" + "testing" + + "github.com/XinFinOrg/XDPoSChain/common" + "github.com/XinFinOrg/XDPoSChain/core/types" + "github.com/XinFinOrg/XDPoSChain/rlp" +) + +// Tests that positional lookup metadata can be stored and retrieved. +func TestLookupStorage(t *testing.T) { + db := NewMemoryDatabase() + + tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11}) + tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22}) + tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33}) + txs := []*types.Transaction{tx1, tx2, tx3} + + block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txs, nil, nil) + + // Check that no transactions entries are in a pristine database + for i, tx := range txs { + if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil { + t.Fatalf("tx #%d [%x]: non existent transaction returned: %v", i, tx.Hash(), txn) + } + } + // Insert all the transactions into the database, and verify contents + WriteBlock(db, block) + WriteTxLookupEntries(db, block) + + for i, tx := range txs { + if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil { + t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash()) + } else { + if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) { + t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i) + } + if tx.Hash() != txn.Hash() { + t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx) + } + } + } + // Delete the transactions and check purge + for i, tx := range txs { + DeleteTxLookupEntry(db, tx.Hash()) + if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil { + t.Fatalf("tx #%d [%x]: deleted transaction returned: %v", i, tx.Hash(), txn) + } + } + // Insert legacy txlookup and verify the data retrieval + for index, tx := range block.Transactions() { + entry := LegacyTxLookupEntry{ + BlockHash: block.Hash(), + BlockIndex: block.NumberU64(), + Index: uint64(index), + } + data, _ := rlp.EncodeToBytes(entry) + db.Put(txLookupKey(tx.Hash()), data) + } + for i, tx := range txs { + if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil { + t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash()) + } else { + if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) { + t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i) + } + if tx.Hash() != txn.Hash() { + t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx) + } + } + } +} diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go new file mode 100644 index 000000000000..fd223fe887d5 --- /dev/null +++ b/core/rawdb/accessors_metadata.go @@ -0,0 +1,37 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "github.com/XinFinOrg/XDPoSChain/ethdb" + "github.com/XinFinOrg/XDPoSChain/rlp" +) + +// ReadDatabaseVersion retrieves the version number of the database. +func ReadDatabaseVersion(db ethdb.KeyValueReader) *uint64 { + var version uint64 + + enc, _ := db.Get(databaseVersionKey) + if len(enc) == 0 { + return nil + } + if err := rlp.DecodeBytes(enc, &version); err != nil { + return nil + } + + return &version +} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 0a08eeed69ed..1153b43fdfb0 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -25,7 +25,12 @@ import ( // The fields below define the low level database schema prefixing. var ( + // databaseVersionKey tracks the current database version. + databaseVersionKey = []byte("DatabaseVersion") + + // headBlockKey tracks the latest known full block's hash. headBlockKey = []byte("LastBlock") + // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header headerHashSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + headerHashSuffix -> hash @@ -33,6 +38,8 @@ var ( blockBodyPrefix = []byte("b") // blockBodyPrefix + num (uint64 big endian) + hash -> block body blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts + + txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata ) const ( @@ -46,6 +53,14 @@ const ( freezerReceiptTable = "receipts" ) +// LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary +// fields. +type LegacyTxLookupEntry struct { + BlockHash common.Hash + BlockIndex uint64 + Index uint64 +} + // encodeBlockNumber encodes a block number as big endian uint64 func encodeBlockNumber(number uint64) []byte { enc := make([]byte, 8) @@ -77,3 +92,8 @@ func blockBodyKey(number uint64, hash common.Hash) []byte { func blockReceiptsKey(number uint64, hash common.Hash) []byte { return append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) } + +// txLookupKey = txLookupPrefix + hash +func txLookupKey(hash common.Hash) []byte { + return append(txLookupPrefix, hash.Bytes()...) +} diff --git a/core/types/log.go b/core/types/log.go index cf1dbc4ecb58..43c71e6ee445 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -69,7 +69,11 @@ type rlpLog struct { Data []byte } -type rlpStorageLog struct { +// rlpStorageLog is the storage encoding of a log. +type rlpStorageLog rlpLog + +// LegacyRlpStorageLog is the previous storage encoding of a log including some redundant fields. +type LegacyRlpStorageLog struct { Address common.Address Topics []common.Hash Data []byte @@ -106,31 +110,34 @@ type LogForStorage Log // EncodeRLP implements rlp.Encoder. func (l *LogForStorage) EncodeRLP(w io.Writer) error { return rlp.Encode(w, rlpStorageLog{ - Address: l.Address, - Topics: l.Topics, - Data: l.Data, - BlockNumber: l.BlockNumber, - TxHash: l.TxHash, - TxIndex: l.TxIndex, - BlockHash: l.BlockHash, - Index: l.Index, + Address: l.Address, + Topics: l.Topics, + Data: l.Data, }) } // DecodeRLP implements rlp.Decoder. +// +// Note some redundant fields(e.g. block number, tx hash etc) will be assembled later. func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error { var dec rlpStorageLog err := s.Decode(&dec) if err == nil { *l = LogForStorage{ - Address: dec.Address, - Topics: dec.Topics, - Data: dec.Data, - BlockNumber: dec.BlockNumber, - TxHash: dec.TxHash, - TxIndex: dec.TxIndex, - BlockHash: dec.BlockHash, - Index: dec.Index, + Address: dec.Address, + Topics: dec.Topics, + Data: dec.Data, + } + } else { + // Try to decode log with previous definition. + var dec LegacyRlpStorageLog + err = s.Decode(&dec) + if err == nil { + *l = LogForStorage{ + Address: dec.Address, + Topics: dec.Topics, + Data: dec.Data, + } } } return err diff --git a/core/types/receipt.go b/core/types/receipt.go index 76507b521064..74236bb101ca 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -111,6 +111,27 @@ type v3StoredReceiptRLP struct { GasUsed uint64 } +// receiptStorageRLP is the storage encoding of a receipt. +type receiptStorageRLP struct { + PostStateOrStatus []byte + CumulativeGasUsed uint64 + TxHash common.Hash + ContractAddress common.Address + Logs []*LogForStorage + GasUsed uint64 +} + +// receiptSwollenStorageRLP is the previous storage encoding of a receipt including some unnecessary fields. +type receiptSwollenStorageRLP struct { + PostStateOrStatus []byte + CumulativeGasUsed uint64 + Bloom Bloom + TxHash common.Hash + ContractAddress common.Address + Logs []*LogForStorage + GasUsed uint64 +} + // NewReceipt creates a barebone transaction receipt, copying the init fields. // Deprecated: create receipts using a struct literal instead. func NewReceipt(root []byte, failed bool, cumulativeGasUsed uint64) *Receipt { diff --git a/eth/backend.go b/eth/backend.go index 45e1bd95ec1a..a03fbb97a63b 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -149,14 +149,13 @@ func New(ctx *node.ServiceContext, config *ethconfig.Config, XDCXServ *XDCx.XDCX if lendingServ != nil { eth.Lending = lendingServ } + bcVersion := core.GetBlockChainVersion(chainDb) log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId) if !config.SkipBcVersionCheck { - bcVersion := core.GetBlockChainVersion(chainDb) if bcVersion != core.BlockChainVersion && bcVersion != 0 { return nil, fmt.Errorf("Blockchain DB version mismatch (%d / %d). Run geth upgradedb.\n", bcVersion, core.BlockChainVersion) } - core.WriteBlockChainVersion(chainDb, core.BlockChainVersion) } var ( vmConfig = vm.Config{EnablePreimageRecording: config.EnablePreimageRecording} diff --git a/les/handler.go b/les/handler.go index 6a4ba688ea3b..5382f9b014f7 100644 --- a/les/handler.go +++ b/les/handler.go @@ -1161,9 +1161,9 @@ func (pm *ProtocolManager) txStatus(hashes []common.Hash) []txStatus { // If the transaction is unknown to the pool, try looking it up locally if stat == core.TxStatusUnknown { - if block, number, index := core.GetTxLookupEntry(pm.chainDb, hashes[i]); block != (common.Hash{}) { + if tx, blockHash, blockNumber, txIndex := rawdb.ReadTransaction(pm.chainDb, hashes[i]); tx != nil { stats[i].Status = core.TxStatusIncluded - stats[i].Lookup = &core.TxLookupEntry{BlockHash: block, BlockIndex: number, Index: index} + stats[i].Lookup = &rawdb.LegacyTxLookupEntry{BlockHash: blockHash, BlockIndex: blockNumber, Index: txIndex} } } } diff --git a/les/handler_test.go b/les/handler_test.go index bac89b5d3353..0c17cf965897 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -556,8 +556,8 @@ func TestTransactionStatusLes2(t *testing.T) { // check if their status is included now block1hash := core.GetCanonicalHash(db, 1) - test(tx1, false, txStatus{Status: core.TxStatusIncluded, Lookup: &core.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 0}}) - test(tx2, false, txStatus{Status: core.TxStatusIncluded, Lookup: &core.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 1}}) + test(tx1, false, txStatus{Status: core.TxStatusIncluded, Lookup: &rawdb.LegacyTxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 0}}) + test(tx2, false, txStatus{Status: core.TxStatusIncluded, Lookup: &rawdb.LegacyTxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 1}}) // create a reorg that rolls them back gchain, _ = core.GenerateChain(params.TestChainConfig, chain.GetBlockByNumber(0), ethash.NewFaker(), db, 2, func(i int, block *core.BlockGen) {}) diff --git a/les/protocol.go b/les/protocol.go index 273ccfcce974..248b31764e46 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -28,6 +28,7 @@ import ( "github.com/XinFinOrg/XDPoSChain/common" "github.com/XinFinOrg/XDPoSChain/core" + "github.com/XinFinOrg/XDPoSChain/core/rawdb" "github.com/XinFinOrg/XDPoSChain/crypto" "github.com/XinFinOrg/XDPoSChain/crypto/secp256k1" "github.com/XinFinOrg/XDPoSChain/rlp" @@ -224,6 +225,6 @@ type proofsData [][]rlp.RawValue type txStatus struct { Status core.TxStatus - Lookup *core.TxLookupEntry `rlp:"nil"` + Lookup *rawdb.LegacyTxLookupEntry `rlp:"nil"` Error string }