From 745b590ced0beb9e7686466aaeffb014ed8d25c8 Mon Sep 17 00:00:00 2001 From: lbtsm <1425930815@qq.com> Date: Mon, 31 Oct 2022 14:59:50 +0800 Subject: [PATCH 01/10] Delete CanonicalNumberToHash when it exceeds the maximum --- chains/ethereum/header_store.go | 36 ++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/chains/ethereum/header_store.go b/chains/ethereum/header_store.go index 77f19f2d..f0a4b278 100644 --- a/chains/ethereum/header_store.go +++ b/chains/ethereum/header_store.go @@ -4,6 +4,8 @@ import ( "bytes" "errors" "fmt" + "sort" + //"github.com/mapprotocol/atlas/core/state" "math/big" "time" @@ -66,16 +68,30 @@ func headerKey(number uint64, hash common.Hash) string { } func (hs *HeaderStore) delOldHeaders() { - //length := len(hs.HeaderNumber) - //log.Info("delOld -------------- length", "length", length, "height", hs.CurNumber) - //if length <= MaxHeaderLimit { - // return - //} - - //delTotal := length - MaxHeaderLimit - //hs.HeaderNumber = hs.HeaderNumber[delTotal:] - //log.Info("before cleaning up the old ethereum headers", "headers length", length) - //log.Info("after cleaning up the old ethereum headers", "headers length", len(hs.HeaderNumber)) + length := len(hs.CanonicalNumberToHash) + log.Info("delOld -------------- length", "length", length, "height", hs.CurNumber) + if length <= MaxHeaderLimit { + return + } + + numbers := make([]uint64, 0, length) + for key := range hs.CanonicalNumberToHash { + numbers = append(numbers, key) + } + + sort.Slice(numbers, func(i, j int) bool { + return numbers[i] < numbers[j] + }) + + delTotal := length - MaxHeaderLimit + for i := 0; i < delTotal; { + number := numbers[i] + delete(hs.CanonicalNumberToHash, number) + log.Info("deleted ethereum canonical header", "number", number) + i++ + } + log.Info("before cleaning up the old canonical ethereum headers", "headers length", length) + log.Info("after cleaning up the old canonical ethereum headers", "headers length", len(hs.CanonicalNumberToHash)) } func encodeHeader(header *Header) []byte { From 7053f96811a011373d0a54038ee1298fa46ad8b0 Mon Sep 17 00:00:00 2001 From: lbtsm <1425930815@qq.com> Date: Mon, 31 Oct 2022 15:18:08 +0800 Subject: [PATCH 02/10] 50000 to 5000 --- chains/ethereum/header_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chains/ethereum/header_store.go b/chains/ethereum/header_store.go index f0a4b278..d9d7c7ac 100644 --- a/chains/ethereum/header_store.go +++ b/chains/ethereum/header_store.go @@ -22,7 +22,7 @@ import ( const ( StoreCacheSize = 20 - MaxHeaderLimit = 50000 + MaxHeaderLimit = 5000 SplicingSymbol = "-" ) From 9a7fec3c8f0f8c923da640050ff400da8980a68d Mon Sep 17 00:00:00 2001 From: lbtsm <1425930815@qq.com> Date: Mon, 31 Oct 2022 15:49:05 +0800 Subject: [PATCH 03/10] doc --- chains/ethereum/header_store.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/chains/ethereum/header_store.go b/chains/ethereum/header_store.go index d9d7c7ac..d594d1ad 100644 --- a/chains/ethereum/header_store.go +++ b/chains/ethereum/header_store.go @@ -114,7 +114,7 @@ func decodeHeader(data []byte, hash common.Hash) *Header { func NewHeaderStore() *HeaderStore { return &HeaderStore{ CanonicalNumberToHash: make(map[uint64]common.Hash), - //HeaderNumber: make([]*big.Int, 0, MaxHeaderLimit), // 数组舍弃多少个，还是通过下标的方式 + //HeaderNumber: make([]*big.Int, 0, MaxHeaderLimit), } } @@ -196,9 +196,9 @@ func (hs *HeaderStore) StoreHeader(state types.StateDB, number uint64, header *L log.Error("Failed to RLP encode HeaderStore", "err", err) return err } - // save 保存到数据库 + // save db state.SetPOWState(address, common.BigToHash(new(big.Int).SetUint64(number)), data) - // 保存到cache中 + // save cache clone, err := cloneLightHeader(header) if err != nil { return err @@ -260,7 +260,6 @@ func (hs *HeaderStore) LoadHeader(number uint64, db types.StateDB) (lh *LightHea }, nil //return nil, errors.New("loadHeader please initialize header store") } - // 先从 lruCache 获取 hash := tools.RlpHash(data) if cc, ok := storeCache.Cache.Get(hash); ok { cp, err := cloneLightHeader(cc.(*LightHeader)) From a7a3767366d2c4c3348c9411e3af72b49f753d33 Mon Sep 17 00:00:00 2001 From: lbtsm <1425930815@qq.com> Date: Tue, 1 Nov 2022 11:13:11 +0800 Subject: [PATCH 04/10] modify header store --- chains/ethereum/header_store.go | 108 ++++++++++++++++------------ chains/ethereum/header_store_ext.go | 76 -------------------- 2 files changed, 61 insertions(+), 123 deletions(-) diff --git a/chains/ethereum/header_store.go b/chains/ethereum/header_store.go index d594d1ad..27176a77 100644 --- a/chains/ethereum/header_store.go +++ b/chains/ethereum/header_store.go @@ -4,9 +4,7 @@ import ( "bytes" "errors" "fmt" - "sort" - - //"github.com/mapprotocol/atlas/core/state" + "math" "math/big" "time" @@ -52,10 +50,10 @@ type Cache struct { } type HeaderStore struct { - CanonicalNumberToHash map[uint64]common.Hash CurNumber uint64 CurHash common.Hash - //HeaderNumber []*big.Int + startNumber uint64 // Record the block number to start synchronization, which is used to obtain the key of the db + CanonicalNumberToHash []common.Hash } type LightHeader struct { @@ -67,31 +65,39 @@ func headerKey(number uint64, hash common.Hash) string { return fmt.Sprintf("%d%s%s", number, SplicingSymbol, hash.Hex()) } -func (hs *HeaderStore) delOldHeaders() { - length := len(hs.CanonicalNumberToHash) - log.Info("delOld -------------- length", "length", length, "height", hs.CurNumber) - if length <= MaxHeaderLimit { - return - } - - numbers := make([]uint64, 0, length) - for key := range hs.CanonicalNumberToHash { - numbers = append(numbers, key) - } +func (hs *HeaderStore) headerIdx(number uint64) uint64 { + return uint64(math.Mod(float64(number), MaxHeaderLimit)) + hs.startNumber +} - sort.Slice(numbers, func(i, j int) bool { - return numbers[i] < numbers[j] - }) +func (hs *HeaderStore) CanonicalHeaderIdx(number uint64) uint64 { + return uint64(math.Mod(float64(number), MaxHeaderLimit)) +} - delTotal := length - MaxHeaderLimit - for i := 0; i < delTotal; { - number := numbers[i] - delete(hs.CanonicalNumberToHash, number) - log.Info("deleted ethereum canonical header", "number", number) - i++ - } - log.Info("before cleaning up the old canonical ethereum headers", "headers length", length) - log.Info("after cleaning up the old canonical ethereum headers", "headers length", len(hs.CanonicalNumberToHash)) +func (hs *HeaderStore) delOldHeaders() { + //length := len(hs.CanonicalNumberToHash) + //log.Info("delOld -------------- length", "length", length, "height", hs.CurNumber) + //if length <= MaxHeaderLimit { + // return + //} + // + //numbers := make([]uint64, 0, length) + //for key := range hs.CanonicalNumberToHash { + // numbers = append(numbers, key) + //} + // + //sort.Slice(numbers, func(i, j int) bool { + // return numbers[i] < numbers[j] + //}) + // + //delTotal := length - MaxHeaderLimit + //for i := 0; i < delTotal; { + // number := numbers[i] + // delete(hs.CanonicalNumberToHash, number) + // log.Info("deleted ethereum canonical header", "number", number) + // i++ + //} + //log.Info("before cleaning up the old canonical ethereum headers", "headers length", length) + //log.Info("after cleaning up the old canonical ethereum headers", "headers length", len(hs.CanonicalNumberToHash)) } func encodeHeader(header *Header) []byte { @@ -113,8 +119,7 @@ func decodeHeader(data []byte, hash common.Hash) *Header { func NewHeaderStore() *HeaderStore { return &HeaderStore{ - CanonicalNumberToHash: make(map[uint64]common.Hash), - //HeaderNumber: make([]*big.Int, 0, MaxHeaderLimit), + CanonicalNumberToHash: make([]common.Hash, MaxHeaderLimit), } } @@ -129,14 +134,14 @@ func (hs *HeaderStore) ResetHeaderStore(state types.StateDB, ethHeaders []byte, number := header.Number.Uint64() h := &HeaderStore{ - CanonicalNumberToHash: map[uint64]common.Hash{ - number: hash, - }, - CurHash: hash, - CurNumber: number, - //HeaderNumber: make([]*big.Int, 0, MaxHeaderLimit), - } - //h.HeaderNumber = append(h.HeaderNumber, header.Number) + CurHash: hash, + CurNumber: number, + startNumber: number, + CanonicalNumberToHash: make([]common.Hash, MaxHeaderLimit), + } + cntIdx := hs.CanonicalHeaderIdx(number) + log.Info("ResetHeaderStore GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) + h.CanonicalNumberToHash[cntIdx] = hash if err := h.Store(state); err != nil { return err } @@ -177,7 +182,6 @@ func (hs *HeaderStore) Store(state types.StateDB) error { return err } - //log.Info("Store save ", "curNumber", hs.CurNumber, "length", len(hs.HeaderNumber)) state.SetPOWState(address, key, data) clone, err := cloneHeaderStore(hs) @@ -190,6 +194,8 @@ func (hs *HeaderStore) Store(state types.StateDB) error { } func (hs *HeaderStore) StoreHeader(state types.StateDB, number uint64, header *LightHeader) error { + idx := hs.headerIdx(number) + log.Info("StoreHeader GetHeaderIdx", "number", number, "idx", idx) address := chains.EthereumHeaderStoreAddress data, err := rlp.EncodeToBytes(header) if err != nil { @@ -197,7 +203,7 @@ func (hs *HeaderStore) StoreHeader(state types.StateDB, number uint64, header *L return err } // save db - state.SetPOWState(address, common.BigToHash(new(big.Int).SetUint64(number)), data) + state.SetPOWState(address, common.BigToHash(new(big.Int).SetUint64(idx)), data) // save cache clone, err := cloneLightHeader(header) if err != nil { @@ -230,7 +236,6 @@ func (hs *HeaderStore) Load(state types.StateDB) (err error) { h = *cp hs.CurHash, hs.CurNumber = h.CurHash, h.CurNumber hs.CanonicalNumberToHash = h.CanonicalNumberToHash - //hs.CanonicalNumberToHash, hs.HeaderNumber = h.CanonicalNumberToHash, h.HeaderNumber return nil } @@ -245,20 +250,20 @@ func (hs *HeaderStore) Load(state types.StateDB) (err error) { } storeCache.Cache.Add(hash, clone) hs.CurHash, hs.CurNumber = h.CurHash, h.CurNumber - //hs.CanonicalNumberToHash, hs.HeaderNumber = h.CanonicalNumberToHash, h.HeaderNumber hs.CanonicalNumberToHash = h.CanonicalNumberToHash return nil } func (hs *HeaderStore) LoadHeader(number uint64, db types.StateDB) (lh *LightHeader, err error) { + idx := hs.headerIdx(number) + log.Info("LoadHeader GetHeaderIdx", "number", number, "idx", idx) address := chains.EthereumHeaderStoreAddress - data := db.GetPOWState(address, common.BigToHash(new(big.Int).SetUint64(number))) + data := db.GetPOWState(address, common.BigToHash(new(big.Int).SetUint64(idx))) if len(data) == 0 { return &LightHeader{ Headers: make(map[string][]byte), TDs: make(map[string]*big.Int), }, nil - //return nil, errors.New("loadHeader please initialize header store") } hash := tools.RlpHash(data) if cc, ok := storeCache.Cache.Get(hash); ok { @@ -314,16 +319,22 @@ func (hs *HeaderStore) HasHeader(hash common.Hash, number uint64, db types.State } func (hs *HeaderStore) ReadCanonicalHash(number uint64) common.Hash { - return hs.CanonicalNumberToHash[number] + cntIdx := hs.CanonicalHeaderIdx(number) + log.Info("ReadCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) + return hs.CanonicalNumberToHash[cntIdx] } func (hs *HeaderStore) WriteCanonicalHash(hash common.Hash, number uint64) { // number -> hash mapping - hs.CanonicalNumberToHash[number] = hash + cntIdx := hs.CanonicalHeaderIdx(number) + log.Info("WriteCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) + hs.CanonicalNumberToHash[cntIdx] = hash } func (hs *HeaderStore) DeleteCanonicalHash(number uint64) { - delete(hs.CanonicalNumberToHash, number) + cntIdx := hs.CanonicalHeaderIdx(number) + log.Info("DeleteCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) + hs.CanonicalNumberToHash[cntIdx] = common.Hash{} } type headerWriteResult struct { @@ -433,6 +444,9 @@ func (hs *HeaderStore) WriteHeaders(db types.StateDB, ethHeaders []byte) (*heade if reorg { if !chainAlreadyCanon { for i := lastNumber + 1; ; i++ { + if i <= hs.CurNumber-MaxHeaderLimit+1 { + continue + } hash := hs.ReadCanonicalHash(i) if hash == (common.Hash{}) { break diff --git a/chains/ethereum/header_store_ext.go b/chains/ethereum/header_store_ext.go index c283339b..72524763 100644 --- a/chains/ethereum/header_store_ext.go +++ b/chains/ethereum/header_store_ext.go @@ -5,85 +5,9 @@ import ( "math/big" "sort" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" ) -type extHeaderStore struct { - Numbers []uint64 - Hashes []common.Hash - //HeadersKey []*big.Int - //HeadersValue [][]byte - //TDsKey []*big.Int - //TDsValue []*big.Int - CurNumber uint64 - CurHash common.Hash -} - -func (hs *HeaderStore) EncodeRLP(w io.Writer) error { - var ( - cl = len(hs.CanonicalNumberToHash) - //hl = len(hs.HeaderNumber) - //tl = len(hs.HeaderNumber) - ) - - var ( - Numbers = make([]uint64, 0, cl) - Hashes = make([]common.Hash, 0, cl) - //HeadersKey = make([]*big.Int, 0, hl) - //TDsKey = make([]*big.Int, 0, tl) - ) - - for number := range hs.CanonicalNumberToHash { - Numbers = append(Numbers, number) - } - sort.Slice(Numbers, func(i, j int) bool { - return Numbers[i] < Numbers[j] - }) - for _, number := range Numbers { - Hashes = append(Hashes, hs.CanonicalNumberToHash[number]) - } - - //for _, k := range hs.HeaderNumber { - // HeadersKey = append(HeadersKey, k) - //} - // - //for _, k := range hs.HeaderNumber { - // TDsKey = append(TDsKey, k) - //} - - return rlp.Encode(w, extHeaderStore{ - Numbers: Numbers, - Hashes: Hashes, - //HeadersKey: HeadersKey, - //TDsKey: TDsKey, - CurNumber: hs.CurNumber, - CurHash: hs.CurHash, - }) -} - -func (hs *HeaderStore) DecodeRLP(s *rlp.Stream) error { - var eh extHeaderStore - if err := s.Decode(&eh); err != nil { - return err - } - - CanonicalNumberToHash := make(map[uint64]common.Hash) - //headerNumber := make([]*big.Int, 0, len(eh.HeadersKey)) - - for i, number := range eh.Numbers { - CanonicalNumberToHash[number] = eh.Hashes[i] - } - //for _, v := range eh.HeadersKey { - // headerNumber = append(headerNumber, v) - //} - - hs.CurNumber, hs.CurHash = eh.CurNumber, eh.CurHash - //hs.CanonicalNumberToHash, hs.HeaderNumber = CanonicalNumberToHash, headerNumber - hs.CanonicalNumberToHash = CanonicalNumberToHash - return nil -} - type extLightHeader struct { HeadersKey []string HeadersValue [][]byte From 34e83bc3c08997ab779c24072ff36eefd813ff3e Mon Sep 17 00:00:00 2001 From: lbtsm <1425930815@qq.com> Date: Tue, 1 Nov 2022 11:40:42 +0800 Subject: [PATCH 05/10] modify header store --- chains/ethereum/header_store.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/chains/ethereum/header_store.go b/chains/ethereum/header_store.go index 27176a77..3a3f2454 100644 --- a/chains/ethereum/header_store.go +++ b/chains/ethereum/header_store.go @@ -25,7 +25,8 @@ const ( ) var ( - storeCache *Cache + storeCache *Cache + startNumber uint64 // Record the block number to start synchronization, which is used to obtain the key of the db ) func init() { @@ -52,7 +53,6 @@ type Cache struct { type HeaderStore struct { CurNumber uint64 CurHash common.Hash - startNumber uint64 // Record the block number to start synchronization, which is used to obtain the key of the db CanonicalNumberToHash []common.Hash } @@ -66,7 +66,7 @@ func headerKey(number uint64, hash common.Hash) string { } func (hs *HeaderStore) headerIdx(number uint64) uint64 { - return uint64(math.Mod(float64(number), MaxHeaderLimit)) + hs.startNumber + return uint64(math.Mod(float64(number), MaxHeaderLimit)) + startNumber } func (hs *HeaderStore) CanonicalHeaderIdx(number uint64) uint64 { @@ -136,7 +136,6 @@ func (hs *HeaderStore) ResetHeaderStore(state types.StateDB, ethHeaders []byte, h := &HeaderStore{ CurHash: hash, CurNumber: number, - startNumber: number, CanonicalNumberToHash: make([]common.Hash, MaxHeaderLimit), } cntIdx := hs.CanonicalHeaderIdx(number) @@ -145,6 +144,7 @@ func (hs *HeaderStore) ResetHeaderStore(state types.StateDB, ethHeaders []byte, if err := h.Store(state); err != nil { return err } + startNumber = number firstHeader := &LightHeader{ Headers: make(map[string][]byte), TDs: make(map[string]*big.Int), From 4a2b5b3432bdf62c0a3c5f14537d410485437197 Mon Sep 17 00:00:00 2001 From: lbtsm <1425930815@qq.com> Date: Tue, 1 Nov 2022 15:50:51 +0800 Subject: [PATCH 06/10] log --- chains/ethereum/header_store.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/chains/ethereum/header_store.go b/chains/ethereum/header_store.go index 3a3f2454..f2f36ec1 100644 --- a/chains/ethereum/header_store.go +++ b/chains/ethereum/header_store.go @@ -139,7 +139,7 @@ func (hs *HeaderStore) ResetHeaderStore(state types.StateDB, ethHeaders []byte, CanonicalNumberToHash: make([]common.Hash, MaxHeaderLimit), } cntIdx := hs.CanonicalHeaderIdx(number) - log.Info("ResetHeaderStore GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) + //log.Info("ResetHeaderStore GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) h.CanonicalNumberToHash[cntIdx] = hash if err := h.Store(state); err != nil { return err @@ -195,7 +195,7 @@ func (hs *HeaderStore) Store(state types.StateDB) error { func (hs *HeaderStore) StoreHeader(state types.StateDB, number uint64, header *LightHeader) error { idx := hs.headerIdx(number) - log.Info("StoreHeader GetHeaderIdx", "number", number, "idx", idx) + //log.Info("StoreHeader GetHeaderIdx", "number", number, "idx", idx) address := chains.EthereumHeaderStoreAddress data, err := rlp.EncodeToBytes(header) if err != nil { @@ -256,7 +256,7 @@ func (hs *HeaderStore) Load(state types.StateDB) (err error) { func (hs *HeaderStore) LoadHeader(number uint64, db types.StateDB) (lh *LightHeader, err error) { idx := hs.headerIdx(number) - log.Info("LoadHeader GetHeaderIdx", "number", number, "idx", idx) + //log.Info("LoadHeader GetHeaderIdx", "number", number, "idx", idx) address := chains.EthereumHeaderStoreAddress data := db.GetPOWState(address, common.BigToHash(new(big.Int).SetUint64(idx))) if len(data) == 0 { @@ -320,20 +320,20 @@ func (hs *HeaderStore) HasHeader(hash common.Hash, number uint64, db types.State func (hs *HeaderStore) ReadCanonicalHash(number uint64) common.Hash { cntIdx := hs.CanonicalHeaderIdx(number) - log.Info("ReadCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) + //log.Info("ReadCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) return hs.CanonicalNumberToHash[cntIdx] } func (hs *HeaderStore) WriteCanonicalHash(hash common.Hash, number uint64) { // number -> hash mapping cntIdx := hs.CanonicalHeaderIdx(number) - log.Info("WriteCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) + //log.Info("WriteCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) hs.CanonicalNumberToHash[cntIdx] = hash } func (hs *HeaderStore) DeleteCanonicalHash(number uint64) { cntIdx := hs.CanonicalHeaderIdx(number) - log.Info("DeleteCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) + //log.Info("DeleteCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) hs.CanonicalNumberToHash[cntIdx] = common.Hash{} } @@ -445,6 +445,7 @@ func (hs *HeaderStore) WriteHeaders(db types.StateDB, ethHeaders []byte) (*heade if !chainAlreadyCanon { for i := lastNumber + 1; ; i++ { if i <= hs.CurNumber-MaxHeaderLimit+1 { + log.Info("chainAlreadyCanon=false, obsolete block", "current", hs.CurNumber, "calNumber", i) continue } hash := hs.ReadCanonicalHash(i) From b32044f45f809a3ba12e54bf83e727460938414f Mon Sep 17 00:00:00 2001 From: lbtsm <1425930815@qq.com> Date: Tue, 1 Nov 2022 15:55:08 +0800 Subject: [PATCH 07/10] 5000 to 100000 --- chains/ethereum/header_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chains/ethereum/header_store.go b/chains/ethereum/header_store.go index f2f36ec1..e558a0c7 100644 --- a/chains/ethereum/header_store.go +++ b/chains/ethereum/header_store.go @@ -20,7 +20,7 @@ import ( const ( StoreCacheSize = 20 - MaxHeaderLimit = 5000 + MaxHeaderLimit = 100000 SplicingSymbol = "-" ) From 515189ef47b8f8c47b06ee4c7627412f616bfc86 Mon Sep 17 00:00:00 2001 From: lbtsm <1425930815@qq.com> Date: Tue, 1 Nov 2022 17:18:13 +0800 Subject: [PATCH 08/10] modify key name --- chains/ethereum/header_store.go | 52 +++++++++------------------------ 1 file changed, 13 insertions(+), 39 deletions(-) diff --git a/chains/ethereum/header_store.go b/chains/ethereum/header_store.go index e558a0c7..148c6f68 100644 --- a/chains/ethereum/header_store.go +++ b/chains/ethereum/header_store.go @@ -65,39 +65,19 @@ func headerKey(number uint64, hash common.Hash) string { return fmt.Sprintf("%d%s%s", number, SplicingSymbol, hash.Hex()) } -func (hs *HeaderStore) headerIdx(number uint64) uint64 { - return uint64(math.Mod(float64(number), MaxHeaderLimit)) + startNumber +func (hs *HeaderStore) headerDbKey(number uint64) common.Hash { + idx := hs.CanonicalHeaderIdx(number) + return common.HexToHash(fmt.Sprintf("%s-%d", "eth2map", idx)) } func (hs *HeaderStore) CanonicalHeaderIdx(number uint64) uint64 { - return uint64(math.Mod(float64(number), MaxHeaderLimit)) + idx := uint64(math.Mod(float64(number), MaxHeaderLimit)) + log.Info("ReadCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", idx) + return idx } func (hs *HeaderStore) delOldHeaders() { - //length := len(hs.CanonicalNumberToHash) - //log.Info("delOld -------------- length", "length", length, "height", hs.CurNumber) - //if length <= MaxHeaderLimit { - // return - //} - // - //numbers := make([]uint64, 0, length) - //for key := range hs.CanonicalNumberToHash { - // numbers = append(numbers, key) - //} - // - //sort.Slice(numbers, func(i, j int) bool { - // return numbers[i] < numbers[j] - //}) - // - //delTotal := length - MaxHeaderLimit - //for i := 0; i < delTotal; { - // number := numbers[i] - // delete(hs.CanonicalNumberToHash, number) - // log.Info("deleted ethereum canonical header", "number", number) - // i++ - //} - //log.Info("before cleaning up the old canonical ethereum headers", "headers length", length) - //log.Info("after cleaning up the old canonical ethereum headers", "headers length", len(hs.CanonicalNumberToHash)) + } func encodeHeader(header *Header) []byte { @@ -139,7 +119,6 @@ func (hs *HeaderStore) ResetHeaderStore(state types.StateDB, ethHeaders []byte, CanonicalNumberToHash: make([]common.Hash, MaxHeaderLimit), } cntIdx := hs.CanonicalHeaderIdx(number) - //log.Info("ResetHeaderStore GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) h.CanonicalNumberToHash[cntIdx] = hash if err := h.Store(state); err != nil { return err @@ -194,16 +173,16 @@ func (hs *HeaderStore) Store(state types.StateDB) error { } func (hs *HeaderStore) StoreHeader(state types.StateDB, number uint64, header *LightHeader) error { - idx := hs.headerIdx(number) - //log.Info("StoreHeader GetHeaderIdx", "number", number, "idx", idx) address := chains.EthereumHeaderStoreAddress data, err := rlp.EncodeToBytes(header) if err != nil { log.Error("Failed to RLP encode HeaderStore", "err", err) return err } + key := hs.headerDbKey(number) + log.Info("StoreHeader GetHeaderKey", "number", number, "key", key.String()) // save db - state.SetPOWState(address, common.BigToHash(new(big.Int).SetUint64(idx)), data) + state.SetPOWState(address, key, data) // save cache clone, err := cloneLightHeader(header) if err != nil { @@ -255,10 +234,10 @@ func (hs *HeaderStore) Load(state types.StateDB) (err error) { } func (hs *HeaderStore) LoadHeader(number uint64, db types.StateDB) (lh *LightHeader, err error) { - idx := hs.headerIdx(number) - //log.Info("LoadHeader GetHeaderIdx", "number", number, "idx", idx) + key := hs.headerDbKey(number) + log.Info("LoadHeader GetHeaderKey", "number", number, "key", key.String()) address := chains.EthereumHeaderStoreAddress - data := db.GetPOWState(address, common.BigToHash(new(big.Int).SetUint64(idx))) + data := db.GetPOWState(address, key) if len(data) == 0 { return &LightHeader{ Headers: make(map[string][]byte), @@ -295,7 +274,6 @@ func (hs *HeaderStore) WriteHeaderAndTd(hash common.Hash, number uint64, td *big } loadHeader.Headers[hash.String()] = encodeHeader(header) loadHeader.TDs[hash.String()] = td - //hs.HeaderNumber = append(hs.HeaderNumber, header.Number) // store return hs.StoreHeader(db, number, loadHeader) } @@ -320,20 +298,16 @@ func (hs *HeaderStore) HasHeader(hash common.Hash, number uint64, db types.State func (hs *HeaderStore) ReadCanonicalHash(number uint64) common.Hash { cntIdx := hs.CanonicalHeaderIdx(number) - //log.Info("ReadCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) return hs.CanonicalNumberToHash[cntIdx] } func (hs *HeaderStore) WriteCanonicalHash(hash common.Hash, number uint64) { - // number -> hash mapping cntIdx := hs.CanonicalHeaderIdx(number) - //log.Info("WriteCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) hs.CanonicalNumberToHash[cntIdx] = hash } func (hs *HeaderStore) DeleteCanonicalHash(number uint64) { cntIdx := hs.CanonicalHeaderIdx(number) - //log.Info("DeleteCanonicalHash GetCanonicalHeaderIdx", "number", number, "idx", cntIdx) hs.CanonicalNumberToHash[cntIdx] = common.Hash{} } From f88c9ed63de8bca1f4746d79bcae83d1b2d2ec79 Mon Sep 17 00:00:00 2001 From: lbtsm <1425930815@qq.com> Date: Tue, 1 Nov 2022 17:36:10 +0800 Subject: [PATCH 09/10] modify key name --- chains/ethereum/header_store.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/chains/ethereum/header_store.go b/chains/ethereum/header_store.go index 148c6f68..e7a9245d 100644 --- a/chains/ethereum/header_store.go +++ b/chains/ethereum/header_store.go @@ -66,8 +66,10 @@ func headerKey(number uint64, hash common.Hash) string { } func (hs *HeaderStore) headerDbKey(number uint64) common.Hash { - idx := hs.CanonicalHeaderIdx(number) - return common.HexToHash(fmt.Sprintf("%s-%d", "eth2map", idx)) + str := fmt.Sprintf("%s-%d", "eth2map", hs.CanonicalHeaderIdx(number)) + key := common.BytesToHash([]byte(str)) + log.Info("StoreHeader GetHeaderKey", "number", number, "str", str, "key", key.String()) + return key } func (hs *HeaderStore) CanonicalHeaderIdx(number uint64) uint64 { @@ -180,7 +182,6 @@ func (hs *HeaderStore) StoreHeader(state types.StateDB, number uint64, header *L return err } key := hs.headerDbKey(number) - log.Info("StoreHeader GetHeaderKey", "number", number, "key", key.String()) // save db state.SetPOWState(address, key, data) // save cache @@ -235,7 +236,6 @@ func (hs *HeaderStore) Load(state types.StateDB) (err error) { func (hs *HeaderStore) LoadHeader(number uint64, db types.StateDB) (lh *LightHeader, err error) { key := hs.headerDbKey(number) - log.Info("LoadHeader GetHeaderKey", "number", number, "key", key.String()) address := chains.EthereumHeaderStoreAddress data := db.GetPOWState(address, key) if len(data) == 0 { From 770956ae277fc5c68cee133641a7a05f5882c325 Mon Sep 17 00:00:00 2001 From: lbtsm <1425930815@qq.com> Date: Wed, 2 Nov 2022 10:54:28 +0800 Subject: [PATCH 10/10] modify slice length --- chains/ethereum/header_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chains/ethereum/header_store.go b/chains/ethereum/header_store.go index e7a9245d..3fab1632 100644 --- a/chains/ethereum/header_store.go +++ b/chains/ethereum/header_store.go @@ -101,7 +101,7 @@ func decodeHeader(data []byte, hash common.Hash) *Header { func NewHeaderStore() *HeaderStore { return &HeaderStore{ - CanonicalNumberToHash: make([]common.Hash, MaxHeaderLimit), + CanonicalNumberToHash: make([]common.Hash, 0), } }