Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/del headers #134

Merged
merged 10 commits into from
Nov 2, 2022
Merged

Feature/del headers #134

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
745b590
Delete CanonicalNumberToHash when it exceeds the maximum
lbtsm Oct 31, 2022
7053f96
50000 to 5000
lbtsm Oct 31, 2022
9a7fec3
doc
lbtsm Oct 31, 2022
a7a3767
modify header store
lbtsm Nov 1, 2022
34e83bc
modify header store
lbtsm Nov 1, 2022
4a2b5b3
log
lbtsm Nov 1, 2022
b32044f
5000 to 100000
lbtsm Nov 1, 2022
515189e
modify key name
lbtsm Nov 1, 2022
f88c9ed
modify key name
lbtsm Nov 1, 2022
770956a
modify slice length
lbtsm Nov 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 62 additions & 32 deletions chains/ethereum/header_store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"bytes"
"errors"
"fmt"
//"github.com/mapprotocol/atlas/core/state"
"math"
"math/big"
"time"

Expand All @@ -20,12 +20,13 @@ import (

const (
StoreCacheSize = 20
MaxHeaderLimit = 50000
MaxHeaderLimit = 100000
SplicingSymbol = "-"
)

var (
storeCache *Cache
storeCache *Cache
startNumber uint64 // Record the block number to start synchronization, which is used to obtain the key of the db
)

func init() {
Expand All @@ -50,10 +51,9 @@ type Cache struct {
}

type HeaderStore struct {
CanonicalNumberToHash map[uint64]common.Hash
CurNumber uint64
CurHash common.Hash
//HeaderNumber []*big.Int
CanonicalNumberToHash []common.Hash
}

type LightHeader struct {
Expand All @@ -65,17 +65,39 @@ func headerKey(number uint64, hash common.Hash) string {
return fmt.Sprintf("%d%s%s", number, SplicingSymbol, hash.Hex())
}

func (hs *HeaderStore) headerIdx(number uint64) uint64 {
return uint64(math.Mod(float64(number), MaxHeaderLimit)) + startNumber
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

startNumber can not be initialized when reboot the node.

Copy link
Collaborator Author

@lbtsm lbtsm Nov 1, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we have made improvements here. We do not use startNumber，Please check the next commit

}

func (hs *HeaderStore) CanonicalHeaderIdx(number uint64) uint64 {
return uint64(math.Mod(float64(number), MaxHeaderLimit))
}

func (hs *HeaderStore) delOldHeaders() {
//length := len(hs.HeaderNumber)
//length := len(hs.CanonicalNumberToHash)
//log.Info("delOld -------------- length", "length", length, "height", hs.CurNumber)
//if length <= MaxHeaderLimit {
// return
//}

//
//numbers := make([]uint64, 0, length)
//for key := range hs.CanonicalNumberToHash {
// numbers = append(numbers, key)
//}
//
//sort.Slice(numbers, func(i, j int) bool {
// return numbers[i] < numbers[j]
//})
//
//delTotal := length - MaxHeaderLimit
//hs.HeaderNumber = hs.HeaderNumber[delTotal:]
//log.Info("before cleaning up the old ethereum headers", "headers length", length)
//log.Info("after cleaning up the old ethereum headers", "headers length", len(hs.HeaderNumber))
//for i := 0; i < delTotal; {
// number := numbers[i]
// delete(hs.CanonicalNumberToHash, number)
// log.Info("deleted ethereum canonical header", "number", number)
// i++
//}
//log.Info("before cleaning up the old canonical ethereum headers", "headers length", length)
//log.Info("after cleaning up the old canonical ethereum headers", "headers length", len(hs.CanonicalNumberToHash))
}

func encodeHeader(header *Header) []byte {
Expand All @@ -97,8 +119,7 @@ func decodeHeader(data []byte, hash common.Hash) *Header {

func NewHeaderStore() *HeaderStore {
return &HeaderStore{
CanonicalNumberToHash: make(map[uint64]common.Hash),
//HeaderNumber: make([]*big.Int, 0, MaxHeaderLimit), // 数组舍弃多少个，还是通过下标的方式
CanonicalNumberToHash: make([]common.Hash, MaxHeaderLimit),
}
}

Expand All @@ -113,17 +134,17 @@ func (hs *HeaderStore) ResetHeaderStore(state types.StateDB, ethHeaders []byte,
number := header.Number.Uint64()

h := &HeaderStore{
CanonicalNumberToHash: map[uint64]common.Hash{
number: hash,
},
CurHash: hash,
CurNumber: number,
//HeaderNumber: make([]*big.Int, 0, MaxHeaderLimit),
}
//h.HeaderNumber = append(h.HeaderNumber, header.Number)
CurHash: hash,
CurNumber: number,
CanonicalNumberToHash: make([]common.Hash, MaxHeaderLimit),
}
cntIdx := hs.CanonicalHeaderIdx(number)
//log.Info("ResetHeaderStore GetCanonicalHeaderIdx", "number", number, "idx", cntIdx)
h.CanonicalNumberToHash[cntIdx] = hash
if err := h.Store(state); err != nil {
return err
}
startNumber = number
firstHeader := &LightHeader{
Headers: make(map[string][]byte),
TDs: make(map[string]*big.Int),
Expand Down Expand Up @@ -161,7 +182,6 @@ func (hs *HeaderStore) Store(state types.StateDB) error {
return err
}

//log.Info("Store save ", "curNumber", hs.CurNumber, "length", len(hs.HeaderNumber))
state.SetPOWState(address, key, data)

clone, err := cloneHeaderStore(hs)
Expand All @@ -174,15 +194,17 @@ func (hs *HeaderStore) Store(state types.StateDB) error {
}

func (hs *HeaderStore) StoreHeader(state types.StateDB, number uint64, header *LightHeader) error {
idx := hs.headerIdx(number)
//log.Info("StoreHeader GetHeaderIdx", "number", number, "idx", idx)
address := chains.EthereumHeaderStoreAddress
data, err := rlp.EncodeToBytes(header)
if err != nil {
log.Error("Failed to RLP encode HeaderStore", "err", err)
return err
}
// save 保存到数据库
state.SetPOWState(address, common.BigToHash(new(big.Int).SetUint64(number)), data)
// 保存到cache中
// save db
state.SetPOWState(address, common.BigToHash(new(big.Int).SetUint64(idx)), data)
// save cache
clone, err := cloneLightHeader(header)
if err != nil {
return err
Expand Down Expand Up @@ -214,7 +236,6 @@ func (hs *HeaderStore) Load(state types.StateDB) (err error) {
h = *cp
hs.CurHash, hs.CurNumber = h.CurHash, h.CurNumber
hs.CanonicalNumberToHash = h.CanonicalNumberToHash
//hs.CanonicalNumberToHash, hs.HeaderNumber = h.CanonicalNumberToHash, h.HeaderNumber
return nil
}

Expand All @@ -229,22 +250,21 @@ func (hs *HeaderStore) Load(state types.StateDB) (err error) {
}
storeCache.Cache.Add(hash, clone)
hs.CurHash, hs.CurNumber = h.CurHash, h.CurNumber
//hs.CanonicalNumberToHash, hs.HeaderNumber = h.CanonicalNumberToHash, h.HeaderNumber
hs.CanonicalNumberToHash = h.CanonicalNumberToHash
return nil
}

func (hs *HeaderStore) LoadHeader(number uint64, db types.StateDB) (lh *LightHeader, err error) {
idx := hs.headerIdx(number)
//log.Info("LoadHeader GetHeaderIdx", "number", number, "idx", idx)
address := chains.EthereumHeaderStoreAddress
data := db.GetPOWState(address, common.BigToHash(new(big.Int).SetUint64(number)))
data := db.GetPOWState(address, common.BigToHash(new(big.Int).SetUint64(idx)))
if len(data) == 0 {
return &LightHeader{
Headers: make(map[string][]byte),
TDs: make(map[string]*big.Int),
}, nil
//return nil, errors.New("loadHeader please initialize header store")
}
// 先从 lruCache 获取
hash := tools.RlpHash(data)
if cc, ok := storeCache.Cache.Get(hash); ok {
cp, err := cloneLightHeader(cc.(*LightHeader))
Expand Down Expand Up @@ -299,16 +319,22 @@ func (hs *HeaderStore) HasHeader(hash common.Hash, number uint64, db types.State
}

func (hs *HeaderStore) ReadCanonicalHash(number uint64) common.Hash {
return hs.CanonicalNumberToHash[number]
cntIdx := hs.CanonicalHeaderIdx(number)
//log.Info("ReadCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", cntIdx)
return hs.CanonicalNumberToHash[cntIdx]
}

func (hs *HeaderStore) WriteCanonicalHash(hash common.Hash, number uint64) {
// number -> hash mapping
hs.CanonicalNumberToHash[number] = hash
cntIdx := hs.CanonicalHeaderIdx(number)
//log.Info("WriteCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", cntIdx)
hs.CanonicalNumberToHash[cntIdx] = hash
}

func (hs *HeaderStore) DeleteCanonicalHash(number uint64) {
delete(hs.CanonicalNumberToHash, number)
cntIdx := hs.CanonicalHeaderIdx(number)
//log.Info("DeleteCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", cntIdx)
hs.CanonicalNumberToHash[cntIdx] = common.Hash{}
}

type headerWriteResult struct {
Expand Down Expand Up @@ -418,6 +444,10 @@ func (hs *HeaderStore) WriteHeaders(db types.StateDB, ethHeaders []byte) (*heade
if reorg {
if !chainAlreadyCanon {
for i := lastNumber + 1; ; i++ {
if i <= hs.CurNumber-MaxHeaderLimit+1 {
log.Info("chainAlreadyCanon=false, obsolete block", "current", hs.CurNumber, "calNumber", i)
continue
}
hash := hs.ReadCanonicalHash(i)
if hash == (common.Hash{}) {
break
Expand Down
76 changes: 0 additions & 76 deletions chains/ethereum/header_store_ext.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,85 +5,9 @@ import (
"math/big"
"sort"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/rlp"
)

type extHeaderStore struct {
Numbers []uint64
Hashes []common.Hash
//HeadersKey []*big.Int
//HeadersValue [][]byte
//TDsKey []*big.Int
//TDsValue []*big.Int
CurNumber uint64
CurHash common.Hash
}

func (hs *HeaderStore) EncodeRLP(w io.Writer) error {
var (
cl = len(hs.CanonicalNumberToHash)
//hl = len(hs.HeaderNumber)
//tl = len(hs.HeaderNumber)
)

var (
Numbers = make([]uint64, 0, cl)
Hashes = make([]common.Hash, 0, cl)
//HeadersKey = make([]*big.Int, 0, hl)
//TDsKey = make([]*big.Int, 0, tl)
)

for number := range hs.CanonicalNumberToHash {
Numbers = append(Numbers, number)
}
sort.Slice(Numbers, func(i, j int) bool {
return Numbers[i] < Numbers[j]
})
for _, number := range Numbers {
Hashes = append(Hashes, hs.CanonicalNumberToHash[number])
}

//for _, k := range hs.HeaderNumber {
// HeadersKey = append(HeadersKey, k)
//}
//
//for _, k := range hs.HeaderNumber {
// TDsKey = append(TDsKey, k)
//}

return rlp.Encode(w, extHeaderStore{
Numbers: Numbers,
Hashes: Hashes,
//HeadersKey: HeadersKey,
//TDsKey: TDsKey,
CurNumber: hs.CurNumber,
CurHash: hs.CurHash,
})
}

func (hs *HeaderStore) DecodeRLP(s *rlp.Stream) error {
var eh extHeaderStore
if err := s.Decode(&eh); err != nil {
return err
}

CanonicalNumberToHash := make(map[uint64]common.Hash)
//headerNumber := make([]*big.Int, 0, len(eh.HeadersKey))

for i, number := range eh.Numbers {
CanonicalNumberToHash[number] = eh.Hashes[i]
}
//for _, v := range eh.HeadersKey {
// headerNumber = append(headerNumber, v)
//}

hs.CurNumber, hs.CurHash = eh.CurNumber, eh.CurHash
//hs.CanonicalNumberToHash, hs.HeaderNumber = CanonicalNumberToHash, headerNumber
hs.CanonicalNumberToHash = CanonicalNumberToHash
return nil
}

type extLightHeader struct {
HeadersKey []string
HeadersValue [][]byte
Expand Down