Upgrade all our opam deps (MinaProtocol#2237)

* upgrade all opam deps + make it compile

* ocamlformat 0.9

* digestif pin is needed to avoid conflict with cohttp digestif usage

* new toolchain

* Bigstring.Stable.V1 is binable

* move to updated snarky master

* snark_params digestif straggler

* macos-setup digestif pin

* work around snarky change

I think MinaProtocol#2044 has the full story here

* fix dune warning

* reformat

* work capacity

* reformat

* sprintf

* revert snarky to a tweaked version of what's on master

* look in the coda branch of the snarky submodule

* bump snarky

* reformat

Author: emberian

.circleci/config.yml

@@ -44,7 +44,7 @@ jobs:
 
     lint:
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive
@@ -120,7 +120,7 @@ jobs:
     build-artifacts--testnet_postake:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         environment:
             DUNE_PROFILE: testnet_postake
         steps:
@@ -156,7 +156,7 @@ jobs:
     build-artifacts--testnet_postake_many_proposers:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         environment:
             DUNE_PROFILE: testnet_postake_many_proposers
         steps:
@@ -192,7 +192,7 @@ jobs:
     build-artifacts--testnet_postake_snarkless_fake_hash:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         environment:
             DUNE_PROFILE: testnet_postake_snarkless_fake_hash
         steps:
@@ -228,7 +228,7 @@ jobs:
     test-unit--test_posig_snarkless:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive
@@ -238,7 +238,7 @@ jobs:
     test-unit--test_postake_snarkless:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive
@@ -248,7 +248,7 @@ jobs:
     test-unit--dev:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive
@@ -258,7 +258,7 @@ jobs:
     test--fake_hash:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive
@@ -270,7 +270,7 @@ jobs:
     test--test_postake:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive
@@ -285,7 +285,7 @@ jobs:
     test--test_postake_bootstrap:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive
@@ -297,7 +297,7 @@ jobs:
     test--test_postake_catchup:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive
@@ -309,7 +309,7 @@ jobs:
     test--test_postake_five_even_snarkless:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive
@@ -321,7 +321,7 @@ jobs:
     test--test_postake_five_even_txns:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive
@@ -333,7 +333,7 @@ jobs:
     test--test_postake_snarkless:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive
@@ -348,7 +348,7 @@ jobs:
     test--test_postake_split:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive
@@ -360,7 +360,7 @@ jobs:
     test--test_postake_split_snarkless:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive
@@ -387,7 +387,7 @@ jobs:
     test--test_postake_txns:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive

.circleci/config.yml.jinja

@@ -44,7 +44,7 @@ jobs:
 
     lint:
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive
@@ -122,7 +122,7 @@ jobs:
     build-artifacts--{{profile}}:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         environment:
             DUNE_PROFILE: {{profile}}
         steps:
@@ -161,7 +161,7 @@ jobs:
     test-unit--{{profile}}:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive
@@ -174,7 +174,7 @@ jobs:
     test--{{profile}}:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+            - image: codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
         steps:
             - checkout
             - run: git submodule sync && git submodule update --init --recursive

.gitmodules

@@ -10,6 +10,12 @@
 [submodule "src/external/ocaml-extlib"]
 	path = src/external/ocaml-extlib
 	url = git@github.com:CodaProtocol/ocaml-extlib.git
+[submodule "src/external/rpc_parallel"]
+	path = src/external/rpc_parallel
+	url = git@github.com:CodaProtocol/rpc_parallel.git
+[submodule "src/external/digestif"]
+	path = src/external/digestif
+	url = git@github.com:CodaProtocol/digestif.git
 [submodule "frontend/wallet/tablecloth"]
 	path = frontend/wallet/tablecloth
 	url = https://github.com/bkase/tablecloth

README-dev.md

@@ -55,7 +55,7 @@ copy of the repo.
 
 * Pull down developer container image  (~2GB download, go stretch your legs)
 
-`docker pull codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5`
+`docker pull codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc`
 
 * Create local builder image
 
@@ -144,6 +144,8 @@ you need, you run `opam switch import src/opam.export`.
 Some of our dependencies aren't taken from `opam`, and aren't integrated
 with `dune`, so you need to add them manually:
 
+* `opam pin add src/external/digestif`
+* `opam pin add src/external/async_kernel`
 * `opam pin add src/external/ocaml-sodium`
 * `opam pin add src/external/rpc_parallel`
 * `opam pin add src/external/ocaml-extlib`

dockerfiles/Dockerfile

@@ -1,4 +1,4 @@
-FROM codaprotocol/coda:toolchain-c4b2cc7c53ef51136350323d52609de5aa83f3a5
+FROM codaprotocol/coda:toolchain-f8dc1642d4ed1d38a7c726f0f2677f53654e4bfc
 
 ENV OPAM_DIR             "/home/opam/.opam/4.07"
 ENV PATH                 "${OPAM_DIR}/bin:$PATH"

dockerfiles/Dockerfile-toolchain

@@ -52,12 +52,16 @@ RUN cd /ocaml-sodium && yes | opam pin add .
 
 # Source copy of rpc_parallel (modified for macOS support)
 ADD /src/external/rpc_parallel /rpc_parallel
-RUN cd /rpc_parallel && yes | opam pin add .
+RUN sudo rm -rf /rpc_parallel/.git && cd /rpc_parallel && yes | opam pin add .
 
 # Source copy of ocaml-extlib (modified to remove module name conflict)
 ADD /src/external/ocaml-extlib /ocaml-extlib
 RUN sudo rm -rf /ocaml-extlib/.git && cd /ocaml-extlib && yes | opam pin add .
 
+# Source copy of digestif (modified to support unpadded SHA256)
+ADD /src/external/digestif /digestif
+RUN sudo rm -rf /digestif/.git && cd /digestif && yes | opam pin add .
+
 ADD /src/external/async_kernel /async_kernel
 RUN sudo rm -rf /async_kernel/.git && cd /async_kernel && yes | opam pin add .
 

docs/style_guide.md

@@ -195,7 +195,7 @@ We use [ppx_inline_test](https://github.com/janestreet/ppx_inline_test) for unit
 
 ```ocaml
 let%test_unit =
-  Quickcheck.test ~sexp:[%sexp_of: Int.t] Int.gen
+  Quickcheck.test ~sexp:[%sexp_of: Int.t] Int.quickcheck_generator
     ~f:(fun x -> assert (Int.equal (f_inv (f x)) x))
 ```
 

scripts/check-snarky-submodule.sh

@@ -7,8 +7,8 @@ cd src/lib/snarky
 CURR=$(git rev-parse HEAD)
 git fetch origin
 
-if ! git rev-list origin/master | grep -q $CURR; then
-  echo "Snarky submodule commit is not an ancestor of snarky/master"
+if ! git rev-list origin/coda | grep -q $CURR; then
+  echo "Snarky submodule commit is not an ancestor of snarky/coda"
   exit 1
 fi
 

scripts/macos-setup.sh

@@ -63,6 +63,7 @@ if [[ $COMPILE_THINGS == "YES" ]]; then
   env TERM=xterm opam pin -y add src/external/ocaml-sodium
   env TERM=xterm opam pin -y add src/external/rpc_parallel
   env TERM=xterm opam pin -y add src/external/ocaml-extlib
+  env TERM=xterm opam pin -y add src/external/digestif
   env TERM=xterm opam pin -y add src/external/async_kernel
   eval $(opam config env)