Merge branch 'master' of github.com:ergoplatform/ergo into weak-blocks

ergoplatform · kushti · Aug 22, 2023 · Sep 12, 2023 · Sep 18, 2023 · Sep 28, 2023
commit 0a9fd56220f385e93bb9fd9edcfbed5ced8abd47
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -21,7 +21,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        scala: [2.13.8, 2.12.10, 2.11.12]
+        scala: [2.13.12, 2.12.18, 2.11.12]
         java: [adopt@1.8]
     runs-on: ${{ matrix.os }}
     steps:
@@ -62,7 +62,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        scala: [2.12.10]
+        scala: [2.12.18]
         java: [adopt@1.8]
     runs-on: ${{ matrix.os }}
     steps:

diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
@@ -0,0 +1,27 @@
+name: Publish release-binaries
+
+on:
+  release:
+    types: [published]
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  release-binaries:
+    name: Publish release binaries
+    runs-on: macos-latest
+    env:
+      ERGO_RELEASE_PLATFORM: macos-x64
+      ERGO_RELEASE_TAG: ${{ github.event.release.tag_name }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+      - name: Download ergo node jar 
+        run: |
+          echo $GITHUB_REF
+          gh release download $ERGO_RELEASE_TAG -p "ergo*"
+      - name: Create release binary files
+        run: python ci/release-binaries.py
+      - name: Put binary files into release
+        run: gh release upload $ERGO_RELEASE_TAG $(echo $(find release -name "ergo-node-*"))
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -36,9 +36,16 @@ jobs:
         env:
           GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
 
-      - name: Publish release 
+      - name: Publish ergo-wallet
         run: sbt +ergoWallet/publishSigned sonatypeBundleRelease
         env:
           PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
           SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
           SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
+
+      - name: Publish ergo-core
+        run: sbt +ergoCore/publishSigned sonatypeBundleRelease
+        env:
+          PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
+          SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
+          SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
diff --git a/Dockerfile b/Dockerfile
@@ -4,9 +4,10 @@ COPY build.sbt findbugs-exclude.xml ./
 COPY project/ project/
 COPY avldb/build.sbt avldb/build.sbt
 COPY avldb/project/ avldb/project/
+COPY ergo-core/build.sbt ergo-core/build.sbt
+COPY ergo-core/project/ ergo-core/project/
 COPY ergo-wallet/build.sbt ergo-wallet/build.sbt
 COPY ergo-wallet/project/ ergo-wallet/project/
-COPY benchmarks/build.sbt benchmarks/build.sbt
 RUN sbt update
 COPY . ./
 RUN sbt assembly

diff --git a/FAQ.md b/FAQ.md
@@ -51,7 +51,7 @@ The details of the Ergo emission schedule and monetary supply can be found in th
 * Telegram: https://t.me/ergoplatform
 
 * Ecosystem: https://sigmaverse.io
-* 
+
 * Github: https://github.com/ergoplatform/ergo
 
 * Documents: https://ergoplatform.org/en/documents/

diff --git a/README.md b/README.md
@@ -76,7 +76,7 @@ Ergo utilizes three types of tests:
 
 1) Unit and property tests: These can be run using the `sbt test` command.
 2) Integration tests: These tests require Docker to be installed. Run them with the `sudo sbt it:test` command.
-3) Bootstrapping tests: These tests are time-consuming as they verify that the node is syncing with the main network in various regimes. Docker is also required for these tests. Run them with the `sudo sbt it2:test` command.
+3) Bootstrapping tests: These tests are time-consuming as they verify that the node is syncing with the main network in various modes. Docker is also required for these tests. Run them with the `sudo sbt it2:test` command.
 
 ## Setting up the Project in an IDE
 
@@ -86,9 +86,17 @@ Ensure that the project can be built with sbt before opening it in an IDE. You m
 
 To open the project in IntelliJ IDEA, select File / Open and navigate to the project folder. This will initiate the Project Import Wizard, which uses the SBT configuration (build.sbt file) to generate the project configuration files for IDEA. You can view the project configuration in the `File / Project Structure...` dialog. If the import is successful, you should be able to compile the project in the IDE. 
 
+## Modules
+
+This repository has modular structure, so only parts which are needed for an application could be used:
+
+* [avldb](avldb/README.md) - implementation of authenticated AVL+ tree used in Ergo, with persistence
+* [ergo-core](ergo-core/README.md) - functionality needed for an SPV client (P2P messages, block section stuctures, PoW, NiPoPoW)
+* ergo-wallet - Java and Scala functionalities to sign and verify transactions
+
 ## Contributing to Ergo
 
-Ergo is an open-source project and we welcome contributions from developers and testers! Join the discussion on [Ergo Discord](https://discord.gg/kj7s7nb) in the #development channel and check out our [Contributing documentation](https://docs.ergoplatform.com/contribute/).
+Ergo is an open-source project and we welcome contributions from developers and testers! Join the discussion over [Ergo Discord](https://discord.gg/kj7s7nb) in #development channel, or Telegram: https://t.me/ErgoDevelopers. Please also check out our [Contributing documentation](https://docs.ergoplatform.com/contribute/).
 
 ## Frequently Asked Questions
 

diff --git a/avldb/build.sbt b/avldb/build.sbt
@@ -1,24 +1,33 @@
 import sbt.Keys.testFrameworks
 
+val scala211 = "2.11.12"
+val scala212 = "2.12.18"
+val scala213 = "2.13.12"
+
 name := "avldb"
 
+val Versions = new {
+
+  val spire = (scalaVersion: String) =>
+    if (scalaVersion == scala213) "0.17.0"
+    else "0.14.1"
+
+  val scalameter = (scalaVersion: String) =>
+    if (scalaVersion == scala213) "0.19"
+    else "0.9"
+}
+
 libraryDependencies ++= Seq(
   "javax.xml.bind" % "jaxb-api" % "2.4.0-b180830.0359",
   "ch.qos.logback" % "logback-classic" % "1.2.3",
   "com.google.guava" % "guava" % "23.0",
-  "org.scorexfoundation" %% "scrypto" % "2.3.0"
-)
-
-libraryDependencies ++= Seq(
+  "org.scorexfoundation" %% "scrypto" % "2.3.0",
   "org.scalatest" %% "scalatest" % "3.1.1" % "test",
   "org.scalacheck" %% "scalacheck" % "1.14.3" % "test",
   "org.scalatestplus" %% "scalatestplus-scalacheck" % "3.1.0.0-RC2" % Test,
-  "com.storm-enroute" %% "scalameter" % "0.9" % "test"
-)
-
-libraryDependencies ++= Seq(
+  "com.storm-enroute" %% "scalameter" % Versions.scalameter(scalaVersion.value) % "test",
   "org.ethereum" % "leveldbjni-all"     % "1.18.3",
-  "org.typelevel" %% "spire" % "0.14.1"
+  "org.typelevel" %% "spire" % Versions.spire(scalaVersion.value)
 )
 
 testOptions in Test := Seq(Tests.Filter(t => !t.matches(".*Benchmark$")))
@@ -38,13 +47,13 @@ publishTo := {
 
 pomIncludeRepository := { _ => false }
 
-scalacOptions ++= Seq("-Xfatal-warnings", "-feature", "-deprecation")
+scalacOptions ++= Seq("-feature", "-deprecation")
 
 // set bytecode version to 8 to fix NoSuchMethodError for various ByteBuffer methods
 // see https://github.com/eclipse/jetty.project/issues/3244
 // these options applied only in "compile" task since scalac crashes on scaladoc compilation with "-release 8"
 // see https://github.com/scala/community-builds/issues/796#issuecomment-423395500
-scalacOptions in(Compile, compile) ++= Seq("-release", "8")
-scalacOptions --= Seq("-Ywarn-numeric-widen", "-Ywarn-value-discard")
+scalacOptions in(Compile, compile) ++= (if (scalaBinaryVersion.value == "2.11") Seq() else Seq("-release", "8"))
+scalacOptions --= Seq("-Ywarn-numeric-widen", "-Ywarn-value-discard", "-Ywarn-unused:params", "-Xfatal-warnings")
 
 enablePlugins(ReproducibleBuildsPlugin)
diff --git a/avldb/project/build.properties b/avldb/project/build.properties
@@ -1 +1 @@
-sbt.version=1.1.1
+sbt.version=1.2.8
diff --git a/avldb/project/plugins.sbt b/avldb/project/plugins.sbt
@@ -2,8 +2,6 @@ logLevel := Level.Warn
 
 addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.1.0")
 
-addSbtPlugin("com.github.tkawachi" % "sbt-lock" % "0.4.0")
-
 addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0")
 
 addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1")

diff --git a/...x/core/serialization/ErgoSerializer.scala → ...atform/serialization/ErgoSerializer.scala b/...x/core/serialization/ErgoSerializer.scala → ...atform/serialization/ErgoSerializer.scala
@@ -1,4 +1,4 @@
-package scorex.core.serialization
+package org.ergoplatform.serialization
 
 import java.nio.ByteBuffer
 import scorex.util.ByteArrayBuilder

diff --git a/...re/serialization/ManifestSerializer.scala → ...rm/serialization/ManifestSerializer.scala b/...re/serialization/ManifestSerializer.scala → ...rm/serialization/ManifestSerializer.scala
@@ -1,4 +1,4 @@
-package scorex.core.serialization
+package org.ergoplatform.serialization
 
 import scorex.crypto.authds.avltree.batch.Constants.DigestType
 import scorex.crypto.authds.avltree.batch.serialization.{BatchAVLProverManifest, ProxyInternalNode}

diff --git a/...ore/serialization/SubtreeSerializer.scala → ...orm/serialization/SubtreeSerializer.scala b/...ore/serialization/SubtreeSerializer.scala → ...orm/serialization/SubtreeSerializer.scala
@@ -1,4 +1,4 @@
-package scorex.core.serialization
+package org.ergoplatform.serialization
 
 import scorex.crypto.authds.avltree.batch.Constants.DigestType
 import scorex.crypto.authds.avltree.batch.{InternalProverNode, ProverLeaf, ProverNodes, VersionedLDBAVLStorage}

diff --git a/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/ProverNodeSerializer.scala b/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/ProverNodeSerializer.scala
@@ -1,7 +1,7 @@
 package scorex.crypto.authds.avltree.batch
 
 import com.google.common.primitives.Ints
-import scorex.core.serialization.ErgoSerializer
+import org.ergoplatform.serialization.ErgoSerializer
 import scorex.crypto.authds.{ADKey, ADValue, Balance}
 import scorex.crypto.authds.avltree.batch.Constants.{DigestType, hashFn}
 import scorex.crypto.authds.avltree.batch.serialization.ProxyInternalNode

diff --git a/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala b/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala
@@ -114,7 +114,7 @@ class VersionedLDBAVLStorage(store: LDBVersionedStore)
       }
 
       def dumpSubtree(sid: DigestType): Try[Unit] = {
-        val builder = mutable.ArrayBuilder.make[Byte]()
+        val builder = new mutable.ArrayBuilder.ofByte
         builder.sizeHint(200000)
         subtreeLoop(sid, builder)
         dumpStorage.insert(sid, builder.result())
@@ -141,7 +141,7 @@ class VersionedLDBAVLStorage(store: LDBVersionedStore)
 
       require(rootNodeLabel.sameElements(expectedRootHash), "Root node hash changed")
 
-      val manifestBuilder = mutable.ArrayBuilder.make[Byte]()
+      val manifestBuilder = new mutable.ArrayBuilder.ofByte
       manifestBuilder.sizeHint(200000)
       manifestBuilder += rootNodeHeight
       manifestBuilder += manifestDepth

diff --git a/avldb/src/main/scala/scorex/db/ByteArrayUtils.scala b/avldb/src/main/scala/scorex/db/ByteArrayUtils.scala
@@ -2,14 +2,14 @@ package scorex.db
 
 object ByteArrayUtils {
 
-  import java.util.Comparator
-
   // Java comparator
-  val BYTE_ARRAY_COMPARATOR: Comparator[Array[Byte]] = (o1: Array[Byte], o2: Array[Byte]) => compare(o1, o2)
-
+  object BYTE_ARRAY_COMPARATOR extends Ordering[Array[Byte]] {
+    def compare(o1: Array[Byte], o2: Array[Byte]) = compare(o1, o2)
+  }
   // Scala comparator
-  implicit val ByteArrayOrdering: Ordering[Array[Byte]] =
-    (o1: Array[Byte], o2: Array[Byte]) => ByteArrayUtils.compare(o1, o2)
+  implicit object ByteArrayOrdering extends Ordering[Array[Byte]] {
+    def compare(o1: Array[Byte], o2: Array[Byte]) = ByteArrayUtils.compare(o1, o2)
+  }
 
   def compare(o1: Array[Byte], o2: Array[Byte]): Int = {
     val len = Math.min(o1.length, o2.length)

diff --git a/avldb/src/main/scala/scorex/db/LDBKVStore.scala b/avldb/src/main/scala/scorex/db/LDBKVStore.scala
@@ -82,7 +82,7 @@ class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging
     breakable {
       while (i.hasNext) {
         val key = i.next().getKey
-        if (ByteArrayUtils.compare(key, last) <= 0) res = Some(key) else break
+        if (ByteArrayUtils.compare(key, last) <= 0) res = Some(key) else break()
       }
     }
     res

diff --git a/...c/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageSpecification.scala b/...c/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageSpecification.scala
@@ -6,7 +6,7 @@ import org.scalatest.Assertion
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.propspec.AnyPropSpec
 import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks
-import scorex.core.serialization.{ManifestSerializer, SubtreeSerializer}
+import org.ergoplatform.serialization.{ManifestSerializer, SubtreeSerializer}
 import scorex.crypto.authds.avltree.batch.helpers.TestHelper
 import scorex.crypto.authds.{ADDigest, ADKey, ADValue, SerializedAdProof}
 import scorex.util.encode.Base16

diff --git a/avldb/src/test/scala/scorex/db/ByteArrayUtilsSpec.scala b/avldb/src/test/scala/scorex/db/ByteArrayUtilsSpec.scala
@@ -6,17 +6,20 @@ import org.scalatest.matchers.should.Matchers
 import org.scalatest.propspec.AnyPropSpec
 import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks
 
+import scala.math.Ordering.Implicits._
+
 class ByteArrayUtilsSpec extends AnyPropSpec with ScalaCheckPropertyChecks with Matchers {
 
   lazy val nonEmptyBytesGen: Gen[Array[Byte]] = Gen.nonEmptyListOf(Arbitrary.arbitrary[Byte])
     .map(_.toArray).suchThat(_.length > 0)
 
   property("compare works properly") {
 
-    //Simple and inefficient way to order byte arrays, based on
-    // https://stackoverflow.com/questions/7109943/how-to-define-orderingarraybyte
-    // but we compare unsigned bytes
-    val ordering: Ordering[Array[Byte]] = Ordering.by((_: Array[Byte]).toIterable.map(_ & 0xFF))
+    val ordering: Ordering[Array[Byte]] =
+      new Ordering[Array[Byte]] {
+        override def compare(o1: Array[Byte], o2: Array[Byte]): Int =
+          implicitly[Ordering[Seq[Int]]].compare(o1.toSeq.map(_ & 0xFF), o2.toSeq.map(_ & 0xFF))
+      }
 
     forAll(nonEmptyBytesGen, nonEmptyBytesGen) { case (bs1, bs2) =>
       val efficientOrdering = Seq(bs1, bs2).sorted(ByteArrayUtils.ByteArrayOrdering)

diff --git a/benchmarks/Jenkinsfile b/benchmarks/Jenkinsfile
diff --git a/benchmarks/build.sbt b/benchmarks/build.sbt
diff --git a/benchmarks/src/test/resources/logback-bench.xml b/benchmarks/src/test/resources/logback-bench.xml