etcm-70-71 added checkpoint to block header + updated header decoder/encoder + added checkpointing config

Author: pslaski

src/main/resources/application.conf

@@ -242,6 +242,7 @@ mantis {
     # If false then that 20% gets burned
     # Doesn't have any effect is ecip1098 is not yet activated
     treasury-opt-out = false
+
   }
 
   # This is the section dedicated to Ethash mining.

src/main/resources/chains/etc-chain.conf

@@ -154,4 +154,7 @@
     "enode://715171f50508aba88aecd1250af392a45a330af91d7b90701c436b618c86aaa1589c9184561907bebbb56439b8f8787bc01f49a7c77276c58c1b09822d75e8e8@52.231.165.108:30303",  // bootnode-azure-koreasouth-001
     "enode://5d6d7cd20d6da4bb83a1d28cadb5d409b64edf314c0335df658c1a54e32c7c4a7ab7823d57c39b6a757556e68ff1df17c748b698544a55cb488b52479a92b60f@104.42.217.25:30303"   // bootnode-azure-westus-001
   ]
+
+  # List of hex encoded public keys of Checkpoint Authorities
+  checkpoint-public-keys = []
 }

src/main/resources/chains/testnet-internal-chain.conf

@@ -127,4 +127,7 @@
 
     # Set of initial nodes
     bootstrap-nodes = []
+
+    # List of hex encoded public keys of Checkpoint Authorities
+    checkpoint-public-keys = []
 }

src/main/scala/io/iohk/ethereum/consensus/ConsensusConfig.scala

@@ -1,13 +1,12 @@
 package io.iohk.ethereum.consensus
 
 import akka.util.ByteString
-import com.typesafe.config.{Config ⇒ TypesafeConfig}
+import com.typesafe.config.{Config => TypesafeConfig}
 import io.iohk.ethereum.consensus.validators.BlockHeaderValidator
 import io.iohk.ethereum.domain.Address
 import io.iohk.ethereum.nodebuilder.ShutdownHookBuilder
 import io.iohk.ethereum.utils.Logger
 
-
 /**
  * Provides generic consensus configuration. Each consensus protocol implementation
  * will use its own specific configuration as well.

src/main/scala/io/iohk/ethereum/crypto/ECDSASignature.scala

@@ -137,3 +137,24 @@ case class ECDSASignature(r: BigInt, s: BigInt, v: Byte) {
   def publicKey(message: ByteString): Option[ByteString] =
     ECDSASignature.recoverPubBytes(r, s, v, None, message.toArray[Byte]).map(ByteString(_))
 }
+
+object ECDSASignatureImplicits {
+
+  import io.iohk.ethereum.rlp.RLPImplicitConversions._
+  import io.iohk.ethereum.rlp.RLPImplicits._
+  import io.iohk.ethereum.rlp._
+
+  implicit val ecdsaSignatureDec: RLPDecoder[ECDSASignature] = new RLPDecoder[ECDSASignature] {
+    override def decode(rlp: RLPEncodeable): ECDSASignature = rlp match {
+      case RLPList(r, s, v) => ECDSASignature(r: ByteString, s: ByteString, v)
+      case _ => throw new RuntimeException("Cannot decode ECDSASignature")
+    }
+  }
+
+  implicit class ECDSASignatureEnc(ecdsaSignature: ECDSASignature) extends RLPSerializable {
+    override def toRLPEncodable: RLPEncodeable = {
+      RLPList(ecdsaSignature.r, ecdsaSignature.s, ecdsaSignature.v)
+    }
+  }
+
+}

src/main/scala/io/iohk/ethereum/db/storage/BlockBodiesStorage.scala

@@ -3,11 +3,11 @@ package io.iohk.ethereum.db.storage
 import java.nio.ByteBuffer
 
 import akka.util.ByteString
-import boopickle.Default.{ Pickle, Unpickle }
+import boopickle.Default.{Pickle, Unpickle}
 import io.iohk.ethereum.crypto.ECDSASignature
 import io.iohk.ethereum.db.dataSource.DataSource
 import io.iohk.ethereum.db.storage.BlockBodiesStorage.BlockBodyHash
-import io.iohk.ethereum.domain.{ Address, BlockHeader, BlockBody, SignedTransaction, Transaction }
+import io.iohk.ethereum.domain.{Address, BlockBody, BlockHeader, Checkpoint, SignedTransaction, Transaction}
 import io.iohk.ethereum.utils.ByteUtils.compactPickledBytes
 
 /**
@@ -38,6 +38,7 @@ object BlockBodiesStorage {
     transformPickler[Address, ByteString](bytes => Address(bytes))(address => address.bytes)
   implicit val transactionPickler: Pickler[Transaction] = generatePickler[Transaction]
   implicit val ecdsaSignaturePickler: Pickler[ECDSASignature] = generatePickler[ECDSASignature]
+  implicit val checkpointPickler: Pickler[Checkpoint] = generatePickler[Checkpoint]
   implicit val signedTransactionPickler: Pickler[SignedTransaction] = transformPickler[SignedTransaction, (Transaction, ECDSASignature)]
   { case (tx, signature) => new SignedTransaction(tx, signature) }{ stx => (stx.tx, stx.signature)}
 

src/main/scala/io/iohk/ethereum/db/storage/BlockHeadersStorage.scala

@@ -3,10 +3,11 @@ package io.iohk.ethereum.db.storage
 import java.nio.ByteBuffer
 
 import akka.util.ByteString
-import boopickle.Default.{ Pickle, Unpickle }
+import boopickle.Default.{Pickle, Unpickle}
+import io.iohk.ethereum.crypto.ECDSASignature
 import io.iohk.ethereum.db.dataSource.DataSource
 import io.iohk.ethereum.db.storage.BlockHeadersStorage.BlockHeaderHash
-import io.iohk.ethereum.domain.BlockHeader
+import io.iohk.ethereum.domain.{BlockHeader, Checkpoint}
 import io.iohk.ethereum.utils.ByteUtils.compactPickledBytes
 
 /**
@@ -31,53 +32,11 @@ class BlockHeadersStorage(val dataSource: DataSource) extends TransactionalKeyVa
 
 object BlockHeadersStorage {
   type BlockHeaderHash = ByteString
-  /** The following types are [[io.iohk.ethereum.domain.BlockHeader]] param types (in exact order).
-    *
-    * Mentioned params:
-    * parentHash, ommersHash, beneficiary, stateRoot, transactionsRoot, receiptsRoot, logsBloom,
-    * difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce.
-    */
-  type BlockHeaderBody = (
-    ByteString,
-    ByteString,
-    ByteString,
-    ByteString,
-    ByteString,
-    ByteString,
-    ByteString,
-    BigInt,
-    BigInt,
-    BigInt,
-    BigInt,
-    Long,
-    ByteString,
-    ByteString,
-    ByteString,
-    Option[Boolean]
-  )
 
   import boopickle.DefaultBasic._
 
   implicit val byteStringPickler: Pickler[ByteString] = transformPickler[ByteString, Array[Byte]](ByteString(_))(_.toArray[Byte])
-  implicit val blockHeaderPickler: Pickler[BlockHeader] = transformPickler[BlockHeader, BlockHeaderBody]
-  { case (ph, oh, b, sr, txr, rr, lb, d, no, gl, gu, ut, ed, mh, n, oo) =>
-    new BlockHeader(ph, oh, b, sr, txr, rr, lb, d, no, gl, gu, ut, ed, mh, n, oo)
-  }{ blockHeader => (
-    blockHeader.parentHash,
-    blockHeader.ommersHash,
-    blockHeader.beneficiary,
-    blockHeader.stateRoot,
-    blockHeader.transactionsRoot,
-    blockHeader.receiptsRoot,
-    blockHeader.logsBloom,
-    blockHeader.difficulty,
-    blockHeader.number,
-    blockHeader.gasLimit,
-    blockHeader.gasUsed,
-    blockHeader.unixTimestamp,
-    blockHeader.extraData,
-    blockHeader.mixHash,
-    blockHeader.nonce,
-    blockHeader.treasuryOptOut
-  )}
+  implicit val ecdsaSignaturePickler: Pickler[ECDSASignature] = generatePickler[ECDSASignature]
+  implicit val checkpointPickler: Pickler[Checkpoint] = generatePickler[Checkpoint]
+  implicit val blockHeaderPickler: Pickler[BlockHeader] = generatePickler[BlockHeader]
 }

src/main/scala/io/iohk/ethereum/domain/BlockBody.scala

@@ -65,7 +65,7 @@ object BlockBody {
       rlpEncodableToBlockBody(
         rlpEncodeable,
         rlp => SignedTransactionRlpEncodableDec(rlp).toSignedTransaction,
-        rlp => BlockheaderEncodableDec(rlp).toBlockHeader
+        rlp => BlockHeaderDec(rlp).toBlockHeader
       )
 
     }

src/main/scala/io/iohk/ethereum/domain/BlockHeader.scala

@@ -2,9 +2,7 @@ package io.iohk.ethereum.domain
 
 import akka.util.ByteString
 import io.iohk.ethereum.crypto.kec256
-import io.iohk.ethereum.rlp.RLPImplicitConversions._
-import io.iohk.ethereum.rlp.RLPImplicits._
-import io.iohk.ethereum.rlp.{RLPEncodeable, RLPList, RLPSerializable, rawDecode, encode => rlpEncode}
+import io.iohk.ethereum.rlp.{RLPDecoder, RLPEncodeable, RLPEncoder, RLPList, RLPSerializable, rawDecode, encode => rlpEncode}
 import org.bouncycastle.util.encoders.Hex
 
 case class BlockHeader(
@@ -23,7 +21,8 @@ case class BlockHeader(
     extraData: ByteString,
     mixHash: ByteString,
     nonce: ByteString,
-    treasuryOptOut: Option[Boolean]) {
+    treasuryOptOut: Option[Boolean],
+    checkpoint: Option[Checkpoint] = None) {
 
   override def toString: String = {
     s"""BlockHeader {
@@ -43,6 +42,7 @@ case class BlockHeader(
        |mixHash: ${Hex.toHexString(mixHash.toArray[Byte])}
        |nonce: ${Hex.toHexString(nonce.toArray[Byte])},
        |treasuryOptOut: $treasuryOptOut
+       |withCheckpoint: ${checkpoint.isDefined}
        |}""".stripMargin
   }
 
@@ -54,72 +54,116 @@ case class BlockHeader(
 
   lazy val hashAsHexString: String = Hex.toHexString(hash.toArray)
 
+  val hasCheckpoint: Boolean = checkpoint.isDefined
+
   def idTag: String =
     s"$number: $hashAsHexString"
 }
 
 object BlockHeader {
 
+  import Checkpoint._
+  import io.iohk.ethereum.rlp.RLPImplicitConversions._
+  import io.iohk.ethereum.rlp.RLPImplicits._
+
+  private implicit val checkpointOptionDecoder = implicitly[RLPDecoder[Option[Checkpoint]]]
+  private implicit val checkpointOptionEncoder = implicitly[RLPEncoder[Option[Checkpoint]]]
+
   val emptyOmmerHash = ByteString(Hex.decode("1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347"))
 
   def getEncodedWithoutNonce(blockHeader: BlockHeader): Array[Byte] = {
     val rlpEncoded = blockHeader.toRLPEncodable match {
-      case rlpList: RLPList if blockHeader.treasuryOptOut.isEmpty =>
-        // Pre ECIP1098 block
-        RLPList(rlpList.items.dropRight(2): _*)
+      case rlpList: RLPList if blockHeader.checkpoint.isDefined =>
+        // post ECIP1098 & ECIP1097 block
+        val rlpItemsWithoutNonce = rlpList.items.dropRight(4) ++ rlpList.items.takeRight(2)
+        RLPList(rlpItemsWithoutNonce: _*)
 
       case rlpList: RLPList if blockHeader.treasuryOptOut.isDefined =>
-        // Post ECIP1098 block
+        // Post ECIP1098 block without checkpoint
         val rlpItemsWithoutNonce = rlpList.items.dropRight(3) :+ rlpList.items.last
         RLPList(rlpItemsWithoutNonce: _*)
 
+      case rlpList: RLPList if blockHeader.treasuryOptOut.isEmpty =>
+        // Pre ECIP1098 & ECIP1097 block
+        RLPList(rlpList.items.dropRight(2): _*)
+
       case _ => throw new Exception("BlockHeader cannot be encoded without nonce and mixHash")
     }
     rlpEncode(rlpEncoded)
   }
 
   implicit class BlockHeaderEnc(blockHeader: BlockHeader) extends RLPSerializable {
+    private def encodeOptOut(definedOptOut: Boolean) = {
+      val encodedOptOut = if(definedOptOut) 1 else 0
+      RLPList(encodedOptOut)
+    }
     override def toRLPEncodable: RLPEncodeable = {
       import blockHeader._
-      treasuryOptOut match {
-        case Some(definedOptOut) =>
-          // Post ECIP1098 block, whole block is encoded
-          val encodedOptOut = if(definedOptOut) 1 else 0
+      (treasuryOptOut, checkpoint) match {
+        case (Some(definedOptOut), Some(_)) =>
+          // Post ECIP1098 & ECIP1097 block, block with treasury enabled and checkpoint is encoded
+          RLPList(parentHash, ommersHash, beneficiary, stateRoot, transactionsRoot, receiptsRoot,
+            logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce, encodeOptOut(definedOptOut), checkpoint)
 
+        case (Some(definedOptOut), None) =>
+          // Post ECIP1098 block, Pre ECIP1097 or without checkpoint, block with treasury enabled is encoded
           RLPList(parentHash, ommersHash, beneficiary, stateRoot, transactionsRoot, receiptsRoot,
-            logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce, RLPList(encodedOptOut))
+            logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce, encodeOptOut(definedOptOut))
 
-        case None =>
-          // Pre ECIP1098 block, encoding works as if optOut field wasn't defined for backwards compatibility
+        case (None, Some(_)) =>
+          // Post ECIP1097 block with checkpoint, treasury disabled, block with checkpoint is encoded
+          RLPList(parentHash, ommersHash, beneficiary, stateRoot, transactionsRoot, receiptsRoot,
+            logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce, RLPList(), checkpoint)
+
+        case _ =>
+          // Pre ECIP1098 and ECIP1097 block, encoding works as if optOut and checkpoint fields weren't defined for backwards compatibility
           RLPList(parentHash, ommersHash, beneficiary, stateRoot, transactionsRoot, receiptsRoot,
             logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce)
       }
     }
   }
 
-  implicit class BlockheaderDec(val bytes: Array[Byte]) extends AnyVal {
-    def toBlockHeader: BlockHeader = BlockheaderEncodableDec(rawDecode(bytes)).toBlockHeader
+  implicit class BlockHeaderByteArrayDec(val bytes: Array[Byte]) extends AnyVal {
+    def toBlockHeader: BlockHeader = BlockHeaderDec(rawDecode(bytes)).toBlockHeader
   }
 
-  implicit class BlockheaderEncodableDec(val rlpEncodeable: RLPEncodeable) extends AnyVal {
+  implicit class BlockHeaderDec(val rlpEncodeable: RLPEncodeable) extends AnyVal {
+    private def decodeOptOut(encodedOptOut: RLPEncodeable): Option[Boolean] = {
+      val booleanOptOut = {
+        if ((encodedOptOut: Int) == 1) true
+        else if ((encodedOptOut: Int) == 0) false
+        else throw new Exception("BlockHeader cannot be decoded with an invalid opt-out")
+      }
+      Some(booleanOptOut)
+    }
     def toBlockHeader: BlockHeader = {
       rlpEncodeable match {
         case RLPList(parentHash, ommersHash, beneficiary, stateRoot, transactionsRoot, receiptsRoot,
-        logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce) =>
-          // Pre ECIP1098 block, encoding works as if optOut field wasn't defined for backwards compatibility
+        logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce, RLPList(encodedOptOut), encodedCheckpoint) =>
+          // Post ECIP1098 & ECIP1097 block with checkpoint, whole block is encoded
+          BlockHeader(parentHash, ommersHash, beneficiary, stateRoot, transactionsRoot, receiptsRoot,
+            logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce,
+            decodeOptOut(encodedOptOut), checkpointOptionDecoder.decode(encodedCheckpoint))
+
+        case RLPList(parentHash, ommersHash, beneficiary, stateRoot, transactionsRoot, receiptsRoot,
+        logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce, RLPList(), encodedCheckpoint) =>
+          // Post ECIP1098 & ECIP1097 block with checkpoint and treasury disabled
           BlockHeader(parentHash, ommersHash, beneficiary, stateRoot, transactionsRoot, receiptsRoot,
-            logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce, None)
+            logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce, None, checkpointOptionDecoder.decode(encodedCheckpoint))
+
 
         case RLPList(parentHash, ommersHash, beneficiary, stateRoot, transactionsRoot, receiptsRoot,
         logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce, RLPList(encodedOptOut)) =>
-          // Post ECIP1098 block, whole block is encoded
-          val booleanOptOut =
-            if ((encodedOptOut: Int) == 1) true
-            else if ((encodedOptOut: Int) == 0) false
-            else throw new Exception("BlockHeader cannot be decoded with an invalid opt-out")
+          // Post ECIP1098 block without checkpoint
+          BlockHeader(parentHash, ommersHash, beneficiary, stateRoot, transactionsRoot, receiptsRoot,
+            logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce, decodeOptOut(encodedOptOut))
 
+
+        case RLPList(parentHash, ommersHash, beneficiary, stateRoot, transactionsRoot, receiptsRoot,
+        logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce) =>
+          // Pre ECIP1098 and ECIP1097 block, decoding works as if optOut and checkpoint fields weren't defined for backwards compatibility
           BlockHeader(parentHash, ommersHash, beneficiary, stateRoot, transactionsRoot, receiptsRoot,
-            logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce, Some(booleanOptOut))
+            logsBloom, difficulty, number, gasLimit, gasUsed, unixTimestamp, extraData, mixHash, nonce, None, None)
 
         case _ =>
           throw new Exception("BlockHeader cannot be decoded")

src/main/scala/io/iohk/ethereum/domain/Checkpoint.scala

@@ -0,0 +1,25 @@
+package io.iohk.ethereum.domain
+
+import io.iohk.ethereum.crypto.ECDSASignature
+import io.iohk.ethereum.rlp._
+
+case class Checkpoint(signatures: Seq[ECDSASignature])
+
+object Checkpoint {
+
+  import io.iohk.ethereum.crypto.ECDSASignatureImplicits._
+
+  implicit val checkpointRLPEncoder: RLPEncoder[Checkpoint] = { checkpoint =>
+    RLPList(checkpoint.signatures.map(_.toRLPEncodable): _*)
+  }
+
+  implicit val checkpointRLPDecoder: RLPDecoder[Checkpoint] = {
+    case signatures: RLPList =>
+      Checkpoint(
+        signatures.items.map(ecdsaSignatureDec.decode)
+      )
+    case _ => throw new RuntimeException("Cannot decode Checkpoint")
+  }
+
+  def empty: Checkpoint = Checkpoint(Nil)
+}

src/main/scala/io/iohk/ethereum/rlp/RLPImplicits.scala

@@ -107,4 +107,15 @@ object RLPImplicits {
       }
   }
 
+  implicit def optionEnc[T](implicit enc: RLPEncoder[T]): RLPEncoder[Option[T]] = {
+    case None => RLPList()
+    case Some(value) => RLPList(enc.encode(value))
+  }
+
+  implicit def optionDec[T](implicit dec: RLPDecoder[T]): RLPDecoder[Option[T]] = {
+    case RLPList(value) => Some(dec.decode(value))
+    case RLPList() => None
+    case rlp => throw RLPException(s"${rlp} should be a list with 1 or 0 elements")
+  }
+
 }