Skip to content

Commit 7dd7354

Browse files
rtkaczykrtkaczyk
committed
[ETCM-263] improve BlockImport wrt checkpoints
1 parent 3b2198e commit 7dd7354

File tree

75 files changed

+918
-869
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

75 files changed

+918
-869
lines changed

‎src/ets/scala/io/iohk/ethereum/ets/blockchain/ScenarioSetup.scala

+4-2
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,12 @@ abstract class ScenarioSetup(_vm: VMImpl, scenario: BlockchainScenario) {
8585
Block(scenario.genesisBlockHeader.toBlockHeader, BlockBody(Nil, Nil))
8686
}
8787

88+
val genesisWeight = ChainWeight.zero.increase(genesisBlock.header)
89+
8890
blockchain
8991
.storeBlock(genesisBlock)
90-
.and(blockchain.storeReceipts(genesisBlock.header.hash, Nil))
91-
.and(blockchain.storeTotalDifficulty(genesisBlock.header.hash, genesisBlock.header.difficulty))
92+
.and(blockchain.storeReceipts(genesisBlock.hash, Nil))
93+
.and(blockchain.storeChainWeight(genesisBlock.hash, genesisWeight))
9294
.commit()
9395

9496
genesisBlock

‎src/it/scala/io/iohk/ethereum/sync/RegularSyncItSpec.scala

+2-2
Original file line numberDiff line numberDiff line change
@@ -84,13 +84,13 @@ class RegularSyncItSpec extends FreeSpecBase with Matchers with BeforeAndAfterAl
8484
_ <- peer2.waitForRegularSyncLoadLastBlock(blockNumer + 3)
8585
} yield {
8686
assert(
87-
peer1.bl.getTotalDifficultyByHash(peer1.bl.getBestBlock().hash) == peer2.bl.getTotalDifficultyByHash(
87+
peer1.bl.getChainWeightByHash(peer1.bl.getBestBlock().hash) == peer2.bl.getChainWeightByHash(
8888
peer2.bl.getBestBlock().hash
8989
)
9090
)
9191
(peer1.bl.getBlockByNumber(blockNumer + 1), peer2.bl.getBlockByNumber(blockNumer + 1)) match {
9292
case (Some(blockP1), Some(blockP2)) =>
93-
assert(peer1.bl.getTotalDifficultyByHash(blockP1.hash) == peer2.bl.getTotalDifficultyByHash(blockP2.hash))
93+
assert(peer1.bl.getChainWeightByHash(blockP1.hash) == peer2.bl.getChainWeightByHash(blockP2.hash))
9494
case (_, _) => fail("invalid difficulty validation")
9595
}
9696
}

‎src/it/scala/io/iohk/ethereum/sync/util/CommonFakePeer.scala

+15-14
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import io.iohk.ethereum.db.components.{RocksDbDataSourceComponent, Storages}
1313
import io.iohk.ethereum.db.dataSource.{RocksDbConfig, RocksDbDataSource}
1414
import io.iohk.ethereum.db.storage.pruning.{ArchivePruning, PruningMode}
1515
import io.iohk.ethereum.db.storage.{AppStateStorage, Namespaces}
16-
import io.iohk.ethereum.domain.{Block, Blockchain, BlockchainImpl}
16+
import io.iohk.ethereum.domain.{Block, Blockchain, BlockchainImpl, ChainWeight}
1717
import io.iohk.ethereum.ledger.InMemoryWorldStateProxy
1818
import io.iohk.ethereum.mpt.MerklePatriciaTrie
1919
import io.iohk.ethereum.network.EtcPeerManagerActor.PeerInfo
@@ -116,8 +116,9 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
116116
Fixtures.Blocks.Genesis.header.copy(stateRoot = ByteString(MerklePatriciaTrie.EmptyRootHash)),
117117
Fixtures.Blocks.Genesis.body
118118
)
119+
val genesisWeight = ChainWeight.zero.increase(genesis.header)
119120

120-
bl.save(genesis, Seq(), genesis.header.difficulty, saveAsBestBlock = true)
121+
bl.save(genesis, Seq(), genesisWeight, saveAsBestBlock = true)
121122

122123
lazy val nh = nodeStatusHolder
123124

@@ -228,15 +229,15 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
228229
)
229230
}
230231

231-
private def broadcastBlock(block: Block, td: BigInt) = {
232-
broadcasterActor ! BroadcastBlock(NewBlock(block, td))
232+
private def broadcastBlock(block: Block, weight: ChainWeight) = {
233+
broadcasterActor ! BroadcastBlock(NewBlock(block, weight))
233234
}
234235

235236
def getCurrentState(): BlockchainState = {
236237
val bestBlock = bl.getBestBlock()
237238
val currentWorldState = getMptForBlock(bestBlock)
238-
val currentTd = bl.getTotalDifficultyByHash(bestBlock.hash).get
239-
BlockchainState(bestBlock, currentWorldState, currentTd)
239+
val currentWeight = bl.getChainWeightByHash(bestBlock.hash).get
240+
BlockchainState(bestBlock, currentWorldState, currentWeight)
240241
}
241242

242243
def startPeer(): Task[Unit] = {
@@ -272,16 +273,16 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
272273
} yield ()
273274
}
274275

275-
private def createChildBlock(parent: Block, parentTd: BigInt, parentWorld: InMemoryWorldStateProxy)(
276+
private def createChildBlock(parent: Block, parentWeight: ChainWeight, parentWorld: InMemoryWorldStateProxy)(
276277
updateWorldForBlock: (BigInt, InMemoryWorldStateProxy) => InMemoryWorldStateProxy
277-
): (Block, BigInt, InMemoryWorldStateProxy) = {
278+
): (Block, ChainWeight, InMemoryWorldStateProxy) = {
278279
val newBlockNumber = parent.header.number + 1
279280
val newWorld = updateWorldForBlock(newBlockNumber, parentWorld)
280281
val newBlock = parent.copy(header =
281282
parent.header.copy(parentHash = parent.header.hash, number = newBlockNumber, stateRoot = newWorld.stateRootHash)
282283
)
283-
val newTd = newBlock.header.difficulty + parentTd
284-
(newBlock, newTd, parentWorld)
284+
val newWeight = parentWeight.increase(newBlock.header)
285+
(newBlock, newWeight, parentWorld)
285286
}
286287

287288
def importBlocksUntil(
@@ -292,12 +293,12 @@ abstract class CommonFakePeer(peerName: String, fakePeerCustomConfig: FakePeerCu
292293
Task(())
293294
} else {
294295
Task {
295-
val currentTd = bl.getTotalDifficultyByHash(block.hash).get
296+
val currentWeight = bl.getChainWeightByHash(block.hash).get
296297
val currentWolrd = getMptForBlock(block)
297-
val (newBlock, newTd, newWorld) = createChildBlock(block, currentTd, currentWolrd)(updateWorldForBlock)
298-
bl.save(newBlock, Seq(), newTd, saveAsBestBlock = true)
298+
val (newBlock, newWeight, _) = createChildBlock(block, currentWeight, currentWolrd)(updateWorldForBlock)
299+
bl.save(newBlock, Seq(), newWeight, saveAsBestBlock = true)
299300
bl.persistCachedNodes()
300-
broadcastBlock(newBlock, newTd)
301+
broadcastBlock(newBlock, newWeight)
301302
}.flatMap(_ => importBlocksUntil(n)(updateWorldForBlock))
302303
}
303304
}

‎src/it/scala/io/iohk/ethereum/sync/util/RegularSyncItSpecUtils.scala

+17-17
Original file line numberDiff line numberDiff line change
@@ -92,12 +92,12 @@ object RegularSyncItSpecUtils {
9292
case None => bl.getBestBlock()
9393
}).flatMap { block =>
9494
Task {
95-
val currentTd = bl
96-
.getTotalDifficultyByHash(block.hash)
97-
.getOrElse(throw new RuntimeException(s"block by hash: ${block.hash} doesn't exist"))
98-
val currentWolrd = getMptForBlock(block)
99-
val (newBlock, newTd, newWorld) = createChildBlock(block, currentTd, currentWolrd)(updateWorldForBlock)
100-
broadcastBlock(newBlock, newTd)
95+
val currentWeight = bl
96+
.getChainWeightByHash(block.hash)
97+
.getOrElse(throw new RuntimeException(s"ChainWeight by hash: ${block.hash} doesn't exist"))
98+
val currentWorld = getMptForBlock(block)
99+
val (newBlock, newWeight, _) = createChildBlock(block, currentWeight, currentWorld)(updateWorldForBlock)
100+
broadcastBlock(newBlock, newWeight)
101101
}
102102
}
103103
}
@@ -110,12 +110,12 @@ object RegularSyncItSpecUtils {
110110
plusDifficulty: BigInt = 0
111111
)(updateWorldForBlock: (BigInt, InMemoryWorldStateProxy) => InMemoryWorldStateProxy): Task[Unit] = Task {
112112
val block: Block = bl.getBestBlock()
113-
val currentTd = bl
114-
.getTotalDifficultyByHash(block.hash)
115-
.getOrElse(throw new RuntimeException(s"block by hash: ${block.hash} doesn't exist"))
113+
val currentWeight = bl
114+
.getChainWeightByHash(block.hash)
115+
.getOrElse(throw new RuntimeException(s"ChainWeight by hash: ${block.hash} doesn't exist"))
116116
val currentWolrd = getMptForBlock(block)
117-
val (newBlock, newTd, newWorld) =
118-
createChildBlock(block, currentTd, currentWolrd, plusDifficulty)(updateWorldForBlock)
117+
val (newBlock, _, _) =
118+
createChildBlock(block, currentWeight, currentWolrd, plusDifficulty)(updateWorldForBlock)
119119
regularSync ! SyncProtocol.MinedBlock(newBlock)
120120
}
121121

@@ -139,18 +139,18 @@ object RegularSyncItSpecUtils {
139139
)
140140
}
141141

142-
private def broadcastBlock(block: Block, td: BigInt) = {
143-
broadcasterActor ! BroadcastBlock(NewBlock(block, td))
142+
private def broadcastBlock(block: Block, weight: ChainWeight) = {
143+
broadcasterActor ! BroadcastBlock(NewBlock(block, weight))
144144
}
145145

146146
private def createChildBlock(
147147
parent: Block,
148-
parentTd: BigInt,
148+
parentWeight: ChainWeight,
149149
parentWorld: InMemoryWorldStateProxy,
150150
plusDifficulty: BigInt = 0
151151
)(
152152
updateWorldForBlock: (BigInt, InMemoryWorldStateProxy) => InMemoryWorldStateProxy
153-
): (Block, BigInt, InMemoryWorldStateProxy) = {
153+
): (Block, ChainWeight, InMemoryWorldStateProxy) = {
154154
val newBlockNumber = parent.header.number + 1
155155
val newWorld = updateWorldForBlock(newBlockNumber, parentWorld)
156156
val newBlock = parent.copy(header =
@@ -161,8 +161,8 @@ object RegularSyncItSpecUtils {
161161
difficulty = plusDifficulty + parent.header.difficulty
162162
)
163163
)
164-
val newTd = newBlock.header.difficulty + parentTd
165-
(newBlock, newTd, parentWorld)
164+
val newWeight = parentWeight.increase(newBlock.header)
165+
(newBlock, newWeight, parentWorld)
166166
}
167167
}
168168

‎src/it/scala/io/iohk/ethereum/sync/util/SyncCommonItSpec.scala

+6-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package io.iohk.ethereum.sync.util
22

33
import java.net.{InetSocketAddress, ServerSocket}
44

5-
import io.iohk.ethereum.domain.Block
5+
import io.iohk.ethereum.domain.{Block, ChainWeight}
66
import io.iohk.ethereum.ledger.InMemoryWorldStateProxy
77

88
object SyncCommonItSpec {
@@ -17,5 +17,9 @@ object SyncCommonItSpec {
1717
}
1818
}
1919

20-
final case class BlockchainState(bestBlock: Block, currentWorldState: InMemoryWorldStateProxy, currentTd: BigInt)
20+
final case class BlockchainState(
21+
bestBlock: Block,
22+
currentWorldState: InMemoryWorldStateProxy,
23+
currentWeight: ChainWeight
24+
)
2125
}

‎src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala

+3-3
Original file line numberDiff line numberDiff line change
@@ -148,13 +148,13 @@ class BlockchainMock(genesisHash: ByteString) extends Blockchain {
148148

149149
override def storeEvmCode(hash: ByteString, evmCode: ByteString): DataSourceBatchUpdate = ???
150150

151-
override def storeTotalDifficulty(blockhash: ByteString, totalDifficulty: BigInt): DataSourceBatchUpdate = ???
151+
override def storeChainWeight(blockhash: ByteString, chainWeight: ChainWeight): DataSourceBatchUpdate = ???
152152

153153
override def saveNode(nodeHash: NodeHash, nodeEncoded: NodeEncoded, blockNumber: BigInt): Unit = ???
154154

155155
override def removeBlock(hash: ByteString, withState: Boolean = true): Unit = ???
156156

157-
override def getTotalDifficultyByHash(blockhash: ByteString): Option[BigInt] = ???
157+
override def getChainWeightByHash(blockhash: ByteString): Option[ChainWeight] = ???
158158

159159
override def getEvmCodeByHash(hash: ByteString): Option[ByteString] = ???
160160

@@ -194,7 +194,7 @@ class BlockchainMock(genesisHash: ByteString) extends Blockchain {
194194

195195
def getBestBlock(): Block = ???
196196

197-
override def save(block: Block, receipts: Seq[Receipt], totalDifficulty: BigInt, saveAsBestBlock: Boolean): Unit = ???
197+
override def save(block: Block, receipts: Seq[Receipt], weight: ChainWeight, saveAsBestBlock: Boolean): Unit = ???
198198

199199
override def getStateStorage: StateStorage = ???
200200

‎src/it/scala/io/iohk/ethereum/txExecTest/util/FixtureProvider.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ object FixtureProvider {
4444
override val blockHeadersStorage: BlockHeadersStorage = new BlockHeadersStorage(dataSource)
4545
override val blockNumberMappingStorage: BlockNumberMappingStorage = new BlockNumberMappingStorage(dataSource)
4646
override val blockBodiesStorage: BlockBodiesStorage = new BlockBodiesStorage(dataSource)
47-
override val totalDifficultyStorage: TotalDifficultyStorage = new TotalDifficultyStorage(dataSource)
47+
override val chainWeightStorage: ChainWeightStorage = new ChainWeightStorage(dataSource)
4848
override val transactionMappingStorage: TransactionMappingStorage = new TransactionMappingStorage(dataSource)
4949
override val nodeStorage: NodeStorage = new NodeStorage(dataSource)
5050
override val cachedNodeStorage: CachedNodeStorage = new CachedNodeStorage(nodeStorage, caches.nodeCache)

‎src/main/scala/io/iohk/ethereum/blockchain/data/GenesisDataLoader.scala

+7-1
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,13 @@ class GenesisDataLoader(blockchain: Blockchain, blockchainConfig: BlockchainConf
118118
case None =>
119119
storage.persist()
120120
stateStorage.forcePersist(GenesisDataLoad)
121-
blockchain.save(Block(header, BlockBody(Nil, Nil)), Nil, header.difficulty, saveAsBestBlock = true)
121+
// TODO: factory for ChainWeight?
122+
blockchain.save(
123+
Block(header, BlockBody(Nil, Nil)),
124+
Nil,
125+
ChainWeight(0, header.difficulty),
126+
saveAsBestBlock = true
127+
)
122128
Success(())
123129
}
124130
}

‎src/main/scala/io/iohk/ethereum/blockchain/sync/BlockBroadcast.scala

+1-2
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,7 @@ class BlockBroadcast(val etcPeerManager: ActorRef, syncConfig: SyncConfig) {
3636

3737
private def shouldSendNewBlock(newBlock: NewBlock, peerInfo: PeerInfo): Boolean =
3838
newBlock.block.header.number > peerInfo.maxBlockNumber ||
39-
newBlock.totalDifficulty > peerInfo.totalDifficulty ||
40-
newBlock.latestCheckpointNumber > peerInfo.latestCheckpointNumber
39+
newBlock.chainWeight > peerInfo.chainWeight
4140

4241
private def broadcastNewBlock(newBlock: NewBlock, peers: Set[Peer]): Unit =
4342
obtainRandomPeerSubset(peers).foreach { peer =>

‎src/main/scala/io/iohk/ethereum/blockchain/sync/FastSync.scala

+15-12
Original file line numberDiff line numberDiff line change
@@ -339,8 +339,8 @@ class FastSync(
339339
val header = headers.head
340340
processHeader(header, peer) match {
341341
case Left(result) => result
342-
case Right(headerAndDif) =>
343-
updateSyncState(headerAndDif._1, headerAndDif._2)
342+
case Right((header, weight)) =>
343+
updateSyncState(header, weight)
344344
if (header.number == syncState.safeDownloadTarget) {
345345
ImportedPivotBlock
346346
} else {
@@ -369,10 +369,10 @@ class FastSync(
369369
}
370370
}
371371

372-
private def updateSyncState(header: BlockHeader, parentTd: BigInt): Unit = {
372+
private def updateSyncState(header: BlockHeader, parentWeight: ChainWeight): Unit = {
373373
blockchain
374374
.storeBlockHeader(header)
375-
.and(blockchain.storeTotalDifficulty(header.hash, parentTd + header.difficulty))
375+
.and(blockchain.storeChainWeight(header.hash, parentWeight.increase(header)))
376376
.commit()
377377

378378
if (header.number > syncState.bestBlockHeaderNumber) {
@@ -389,14 +389,17 @@ class FastSync(
389389
syncState = syncState.updateNextBlockToValidate(header, K, X)
390390
}
391391

392-
private def processHeader(header: BlockHeader, peer: Peer): Either[HeaderProcessingResult, (BlockHeader, BigInt)] =
392+
private def processHeader(
393+
header: BlockHeader,
394+
peer: Peer
395+
): Either[HeaderProcessingResult, (BlockHeader, ChainWeight)] =
393396
for {
394397
validatedHeader <- validateHeader(header, peer)
395-
parentDifficulty <- getParentDifficulty(header)
396-
} yield (validatedHeader, parentDifficulty)
398+
parentWeight <- getParentChainWeight(header)
399+
} yield (validatedHeader, parentWeight)
397400

398-
private def getParentDifficulty(header: BlockHeader) = {
399-
blockchain.getTotalDifficultyByHash(header.parentHash).toRight(ParentDifficultyNotFound(header))
401+
private def getParentChainWeight(header: BlockHeader) = {
402+
blockchain.getChainWeightByHash(header.parentHash).toRight(ParentChainWeightNotFound(header))
400403
}
401404

402405
private def handleRewind(header: BlockHeader, peer: Peer, N: Int): Unit = {
@@ -417,11 +420,11 @@ class FastSync(
417420
private def handleBlockHeaders(peer: Peer, headers: Seq[BlockHeader]) = {
418421
if (checkHeadersChain(headers)) {
419422
processHeaders(peer, headers) match {
420-
case ParentDifficultyNotFound(header) =>
423+
case ParentChainWeightNotFound(header) =>
421424
// We could end in wrong fork and get blocked so we should rewind our state a little
422425
// we blacklist peer just in case we got malicious peer which would send us bad blocks, forcing us to rollback
423426
// to genesis
424-
log.warning("Parent difficulty not found for block {}, not processing rest of headers", header.idTag)
427+
log.warning("Parent chain weight not found for block {}, not processing rest of headers", header.idTag)
425428
handleRewind(header, peer, syncConfig.fastSyncBlockValidationN)
426429
case HeadersProcessingFinished =>
427430
processSyncing()
@@ -888,7 +891,7 @@ object FastSync {
888891

889892
sealed abstract class HeaderProcessingResult
890893
case object HeadersProcessingFinished extends HeaderProcessingResult
891-
case class ParentDifficultyNotFound(header: BlockHeader) extends HeaderProcessingResult
894+
case class ParentChainWeightNotFound(header: BlockHeader) extends HeaderProcessingResult
892895
case class ValidationFailed(header: BlockHeader, peer: Peer) extends HeaderProcessingResult
893896
case object ImportedPivotBlock extends HeaderProcessingResult
894897

‎src/main/scala/io/iohk/ethereum/blockchain/sync/PeersClient.scala

+3-3
Original file line numberDiff line numberDiff line change
@@ -161,12 +161,12 @@ object PeersClient {
161161

162162
def bestPeer(peersToDownloadFrom: Map[Peer, PeerInfo]): Option[Peer] = {
163163
val peersToUse = peersToDownloadFrom
164-
.collect { case (ref, PeerInfo(_, totalDifficulty, latestChkp, true, _, _)) =>
165-
(ref, totalDifficulty, latestChkp)
164+
.collect { case (ref, PeerInfo(_, chainWeight, true, _, _)) =>
165+
(ref, chainWeight)
166166
}
167167

168168
if (peersToUse.nonEmpty) {
169-
val (peer, _, _) = peersToUse.maxBy { case (_, td, latestChkp) => latestChkp -> td }
169+
val (peer, _) = peersToUse.maxBy(_._2)
170170
Some(peer)
171171
} else {
172172
None

‎src/main/scala/io/iohk/ethereum/blockchain/sync/PivotBlockSelector.scala

+2-3
Original file line numberDiff line numberDiff line change
@@ -173,9 +173,8 @@ class PivotBlockSelector(
173173
}
174174

175175
private def collectVoters: ElectionDetails = {
176-
val peersUsedToChooseTarget = peersToDownloadFrom.collect {
177-
case (peer, PeerInfo(_, _, _, true, maxBlockNumber, _)) =>
178-
(peer, maxBlockNumber)
176+
val peersUsedToChooseTarget = peersToDownloadFrom.collect { case (peer, PeerInfo(_, _, true, maxBlockNumber, _)) =>
177+
(peer, maxBlockNumber)
179178
}
180179

181180
val peersSortedByBestNumber = peersUsedToChooseTarget.toList.sortBy { case (_, number) => -number }

‎src/main/scala/io/iohk/ethereum/blockchain/sync/SyncStateSchedulerActor.scala

+2-1
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,8 @@ class SyncStateSchedulerActor(downloader: ActorRef, sync: SyncStateScheduler, sy
5454
val initStats = ProcessingStatistics().addSaved(result.writtenElements)
5555
val initState = startSyncing(startSignal.stateRoot, startSignal.blockNumber)
5656
context become (syncing(initState, initStats, startSignal.blockNumber, sender))
57-
case Some((restartSignal: RestartRequested.type, sender)) =>
57+
case Some((RestartRequested, sender)) =>
58+
// TODO: are we testing this path?
5859
sender ! WaitingForNewTargetBlock
5960
context.become(idle(ProcessingStatistics().addSaved(result.writtenElements)))
6061
case _ =>

‎src/main/scala/io/iohk/ethereum/blockchain/sync/regular/BlockFetcher.scala

+1-1
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@ class BlockFetcher(
192192
supervisor ! ProgressProtocol.GotNewBlock(newState.knownTop)
193193

194194
fetchBlocks(newState)
195-
case MessageFromPeer(NewBlock(block, _, _), peerId) =>
195+
case MessageFromPeer(NewBlock(_, block, _), peerId) =>
196196
val newBlockNr = block.number
197197
val nextExpectedBlock = state.lastFullBlockNumber + 1
198198

0 commit comments

Comments
 (0)