Skip to content

Commit

Permalink
Merge pull request #134 from lbtsm/feature/del-headers
Browse files Browse the repository at this point in the history 
Feature/del headers
  • Loading branch information
@mapdev33
mapdev33 authored Nov 2, 2022
2 parents 54a2f42 + 770956a commit 70367c5
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 114 deletions.
80 changes: 42 additions & 38 deletions chains/ethereum/header_store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"bytes"
"errors"
"fmt"
//"github.com/mapprotocol/atlas/core/state"
"math"
"math/big"
"time"

Expand All @@ -20,12 +20,13 @@ import (

const (
StoreCacheSize = 20
MaxHeaderLimit = 50000
MaxHeaderLimit = 100000
SplicingSymbol = "-"
)

var (
storeCache *Cache
storeCache *Cache
startNumber uint64 // Record the block number to start synchronization, which is used to obtain the key of the db
)

func init() {
Expand All @@ -50,10 +51,9 @@ type Cache struct {
}

type HeaderStore struct {
CanonicalNumberToHash map[uint64]common.Hash
CurNumber uint64
CurHash common.Hash
//HeaderNumber []*big.Int
CanonicalNumberToHash []common.Hash
}

type LightHeader struct {
Expand All @@ -65,17 +65,21 @@ func headerKey(number uint64, hash common.Hash) string {
return fmt.Sprintf("%d%s%s", number, SplicingSymbol, hash.Hex())
}

func (hs *HeaderStore) headerDbKey(number uint64) common.Hash {
str := fmt.Sprintf("%s-%d", "eth2map", hs.CanonicalHeaderIdx(number))
key := common.BytesToHash([]byte(str))
log.Info("StoreHeader GetHeaderKey", "number", number, "str", str, "key", key.String())
return key
}

func (hs *HeaderStore) CanonicalHeaderIdx(number uint64) uint64 {
idx := uint64(math.Mod(float64(number), MaxHeaderLimit))
log.Info("ReadCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", idx)
return idx
}

func (hs *HeaderStore) delOldHeaders() {
//length := len(hs.HeaderNumber)
//log.Info("delOld -------------- length", "length", length, "height", hs.CurNumber)
//if length <= MaxHeaderLimit {
// return
//}

//delTotal := length - MaxHeaderLimit
//hs.HeaderNumber = hs.HeaderNumber[delTotal:]
//log.Info("before cleaning up the old ethereum headers", "headers length", length)
//log.Info("after cleaning up the old ethereum headers", "headers length", len(hs.HeaderNumber))
}

func encodeHeader(header *Header) []byte {
Expand All @@ -97,8 +101,7 @@ func decodeHeader(data []byte, hash common.Hash) *Header {

func NewHeaderStore() *HeaderStore {
return &HeaderStore{
CanonicalNumberToHash: make(map[uint64]common.Hash),
//HeaderNumber: make([]*big.Int, 0, MaxHeaderLimit), // 数组舍弃多少个，还是通过下标的方式
CanonicalNumberToHash: make([]common.Hash, 0),
}
}

Expand All @@ -113,17 +116,16 @@ func (hs *HeaderStore) ResetHeaderStore(state types.StateDB, ethHeaders []byte,
number := header.Number.Uint64()

h := &HeaderStore{
CanonicalNumberToHash: map[uint64]common.Hash{
number: hash,
},
CurHash: hash,
CurNumber: number,
//HeaderNumber: make([]*big.Int, 0, MaxHeaderLimit),
}
//h.HeaderNumber = append(h.HeaderNumber, header.Number)
CurHash: hash,
CurNumber: number,
CanonicalNumberToHash: make([]common.Hash, MaxHeaderLimit),
}
cntIdx := hs.CanonicalHeaderIdx(number)
h.CanonicalNumberToHash[cntIdx] = hash
if err := h.Store(state); err != nil {
return err
}
startNumber = number
firstHeader := &LightHeader{
Headers: make(map[string][]byte),
TDs: make(map[string]*big.Int),
Expand Down Expand Up @@ -161,7 +163,6 @@ func (hs *HeaderStore) Store(state types.StateDB) error {
return err
}

//log.Info("Store save ", "curNumber", hs.CurNumber, "length", len(hs.HeaderNumber))
state.SetPOWState(address, key, data)

clone, err := cloneHeaderStore(hs)
Expand All @@ -180,9 +181,10 @@ func (hs *HeaderStore) StoreHeader(state types.StateDB, number uint64, header *L
log.Error("Failed to RLP encode HeaderStore", "err", err)
return err
}
// save 保存到数据库
state.SetPOWState(address, common.BigToHash(new(big.Int).SetUint64(number)), data)
// 保存到cache中
key := hs.headerDbKey(number)
// save db
state.SetPOWState(address, key, data)
// save cache
clone, err := cloneLightHeader(header)
if err != nil {
return err
Expand Down Expand Up @@ -214,7 +216,6 @@ func (hs *HeaderStore) Load(state types.StateDB) (err error) {
h = *cp
hs.CurHash, hs.CurNumber = h.CurHash, h.CurNumber
hs.CanonicalNumberToHash = h.CanonicalNumberToHash
//hs.CanonicalNumberToHash, hs.HeaderNumber = h.CanonicalNumberToHash, h.HeaderNumber
return nil
}

Expand All @@ -229,22 +230,20 @@ func (hs *HeaderStore) Load(state types.StateDB) (err error) {
}
storeCache.Cache.Add(hash, clone)
hs.CurHash, hs.CurNumber = h.CurHash, h.CurNumber
//hs.CanonicalNumberToHash, hs.HeaderNumber = h.CanonicalNumberToHash, h.HeaderNumber
hs.CanonicalNumberToHash = h.CanonicalNumberToHash
return nil
}

func (hs *HeaderStore) LoadHeader(number uint64, db types.StateDB) (lh *LightHeader, err error) {
key := hs.headerDbKey(number)
address := chains.EthereumHeaderStoreAddress
data := db.GetPOWState(address, common.BigToHash(new(big.Int).SetUint64(number)))
data := db.GetPOWState(address, key)
if len(data) == 0 {
return &LightHeader{
Headers: make(map[string][]byte),
TDs: make(map[string]*big.Int),
}, nil
//return nil, errors.New("loadHeader please initialize header store")
}
// 先从 lruCache 获取
hash := tools.RlpHash(data)
if cc, ok := storeCache.Cache.Get(hash); ok {
cp, err := cloneLightHeader(cc.(*LightHeader))
Expand Down Expand Up @@ -275,7 +274,6 @@ func (hs *HeaderStore) WriteHeaderAndTd(hash common.Hash, number uint64, td *big
}
loadHeader.Headers[hash.String()] = encodeHeader(header)
loadHeader.TDs[hash.String()] = td
//hs.HeaderNumber = append(hs.HeaderNumber, header.Number)
// store
return hs.StoreHeader(db, number, loadHeader)
}
Expand All @@ -299,16 +297,18 @@ func (hs *HeaderStore) HasHeader(hash common.Hash, number uint64, db types.State
}

func (hs *HeaderStore) ReadCanonicalHash(number uint64) common.Hash {
return hs.CanonicalNumberToHash[number]
cntIdx := hs.CanonicalHeaderIdx(number)
return hs.CanonicalNumberToHash[cntIdx]
}

func (hs *HeaderStore) WriteCanonicalHash(hash common.Hash, number uint64) {
// number -> hash mapping
hs.CanonicalNumberToHash[number] = hash
cntIdx := hs.CanonicalHeaderIdx(number)
hs.CanonicalNumberToHash[cntIdx] = hash
}

func (hs *HeaderStore) DeleteCanonicalHash(number uint64) {
delete(hs.CanonicalNumberToHash, number)
cntIdx := hs.CanonicalHeaderIdx(number)
hs.CanonicalNumberToHash[cntIdx] = common.Hash{}
}

type headerWriteResult struct {
Expand Down Expand Up @@ -418,6 +418,10 @@ func (hs *HeaderStore) WriteHeaders(db types.StateDB, ethHeaders []byte) (*heade
if reorg {
if !chainAlreadyCanon {
for i := lastNumber + 1; ; i++ {
if i <= hs.CurNumber-MaxHeaderLimit+1 {
log.Info("chainAlreadyCanon=false, obsolete block", "current", hs.CurNumber, "calNumber", i)
continue
}
hash := hs.ReadCanonicalHash(i)
if hash == (common.Hash{}) {
break
Expand Down
76 changes: 0 additions & 76 deletions chains/ethereum/header_store_ext.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,85 +5,9 @@ import (
"math/big"
"sort"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/rlp"
)

type extHeaderStore struct {
Numbers []uint64
Hashes []common.Hash
//HeadersKey []*big.Int
//HeadersValue [][]byte
//TDsKey []*big.Int
//TDsValue []*big.Int
CurNumber uint64
CurHash common.Hash
}

func (hs *HeaderStore) EncodeRLP(w io.Writer) error {
var (
cl = len(hs.CanonicalNumberToHash)
//hl = len(hs.HeaderNumber)
//tl = len(hs.HeaderNumber)
)

var (
Numbers = make([]uint64, 0, cl)
Hashes = make([]common.Hash, 0, cl)
//HeadersKey = make([]*big.Int, 0, hl)
//TDsKey = make([]*big.Int, 0, tl)
)

for number := range hs.CanonicalNumberToHash {
Numbers = append(Numbers, number)
}
sort.Slice(Numbers, func(i, j int) bool {
return Numbers[i] < Numbers[j]
})
for _, number := range Numbers {
Hashes = append(Hashes, hs.CanonicalNumberToHash[number])
}

//for _, k := range hs.HeaderNumber {
// HeadersKey = append(HeadersKey, k)
//}
//
//for _, k := range hs.HeaderNumber {
// TDsKey = append(TDsKey, k)
//}

return rlp.Encode(w, extHeaderStore{
Numbers: Numbers,
Hashes: Hashes,
//HeadersKey: HeadersKey,
//TDsKey: TDsKey,
CurNumber: hs.CurNumber,
CurHash: hs.CurHash,
})
}

func (hs *HeaderStore) DecodeRLP(s *rlp.Stream) error {
var eh extHeaderStore
if err := s.Decode(&eh); err != nil {
return err
}

CanonicalNumberToHash := make(map[uint64]common.Hash)
//headerNumber := make([]*big.Int, 0, len(eh.HeadersKey))

for i, number := range eh.Numbers {
CanonicalNumberToHash[number] = eh.Hashes[i]
}
//for _, v := range eh.HeadersKey {
// headerNumber = append(headerNumber, v)
//}

hs.CurNumber, hs.CurHash = eh.CurNumber, eh.CurHash
//hs.CanonicalNumberToHash, hs.HeaderNumber = CanonicalNumberToHash, headerNumber
hs.CanonicalNumberToHash = CanonicalNumberToHash
return nil
}

type extLightHeader struct {
HeadersKey []string
HeadersValue [][]byte
Expand Down

0 comments on commit 70367c5

Please sign in to comment.