Merge branch 'develop' into feature/partial_tree

libzkbob-rs-wasm/Cargo.toml

@@ -15,6 +15,7 @@ default = ["console_error_panic_hook"]
 bundler = ["libzkbob-rs/web"]
 web = ["libzkbob-rs/web"]
 nodejs = ["libzkbob-rs/node"]
+multicore = ["fawkes-crypto/multicore", "wasm-bindgen-rayon"]
 
 [dependencies]
 wasm-bindgen = { version = "0.2.74" }
@@ -35,7 +36,7 @@ wee_alloc = { version = "0.4.5", optional = true }
 libzkbob-rs = { path = "../libzkbob-rs" }
 libzeropool = { git = "https://github.com/zkBob/libzeropool", branch = "multicore-wasm", version = "0.5.6", default-features = false }
 getrandom = { version = "0.2.3", features = ["js"] }
-fawkes-crypto = { git = "https://github.com/zkBob/fawkes-crypto", branch = "multicore-wasm", version = "4.2.4", features = ["wasm", "multicore", "serde_support"] }
+fawkes-crypto = { git = "https://github.com/zkBob/fawkes-crypto", branch = "multicore-wasm", version = "4.2.4", features = ["wasm", "serde_support"] }
 bs58 = "0.4.0"
 kvdb-web = { version = "0.9.0", path = "../libs/kvdb-web"}
 kvdb = "0.9.0"
@@ -49,8 +50,10 @@ serde-wasm-bindgen = "0.3.0"
 lazy_static = "1.4.0"
 hex = { version = "0.4.3", features = ["serde"] }
 js-sys = "0.3.55"
-wasm-bindgen-rayon = "1.0"
+wasm-bindgen-rayon = { version = "1.0", optional = true }
 rayon = "1.5.1"
+bincode = "1.3.3"
+
 
 [dev-dependencies]
 wasm-bindgen-test = "0.3.24"
@@ -72,3 +75,4 @@ wasm-opt = false
 
 [package.metadata.wasm-pack.profile.release]
 wasm-opt = false
+

libzkbob-rs-wasm/scripts/build

@@ -1,32 +1,52 @@
 #!/usr/bin/env bash
 
+set -e
+
+RUSTUP_MT_TOOLCHAIN=nightly-2022-12-11
+RUSTUP_ST_TOOLCHAIN=stable-2022-11-03
 PARENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd )"
 
+if ! command -v jq &> /dev/null
+then
+    echo "jq is required to run this script: https://stedolan.github.io/jq/"
+    exit
+fi
+
+function patch_package_json () {
+  sed -i.bak -E "s/\"name\": \"libzkbob-rs-wasm\"/\"name\": \"libzkbob-rs-wasm-$1\"/g" $PARENT_DIR/$1/package.json
+  jq '.files |= . + ["snippets"]' $PARENT_DIR/$1/package.json > $PARENT_DIR/$1/tmp.json \
+    && mv $PARENT_DIR/$1/tmp.json $PARENT_DIR/$1/package.json
+}
+
+# build $name $features $is_mt
 function build () {
-  wasm-pack build --release --target $1 -d $1 -- --features $1
+  if [ "$3" = true ] ; then	
+    RUSTFLAGS='-C target-feature=+atomics,+bulk-memory,+mutable-globals' \
+      rustup run $RUSTUP_MT_TOOLCHAIN \
+      wasm-pack build --release --target web -d $1 \
+      -- --features $2 -Z build-std=panic_abort,std	
+  else
+    rustup run $RUSTUP_ST_TOOLCHAIN \
+      wasm-pack build --release --target web -d $1 -- --features $2	
+  fi
+
   # Optimize the binary, since wasm-pack refuses to see wasm-opt
-  wasm-opt -o $PARENT_DIR/$1/wasm-opt.wasm -O3 $PARENT_DIR/$1/libzkbob_rs_wasm_bg.wasm && mv $PARENT_DIR/$1/wasm-opt.wasm $PARENT_DIR/$1/libzkbob_rs_wasm_bg.wasm
-  # Patch typings for async methods
-  sed -i.bak -E 's/(createDeposit\(.*\)):.*$/\1: Promise<TransactionData>;/g' $PARENT_DIR/$1/libzkbob_rs_wasm.d.ts
-  sed -i.bak -E 's/(createDepositPermittable\(.*\)):.*$/\1: Promise<TransactionData>;/g' $PARENT_DIR/$1/libzkbob_rs_wasm.d.ts
-  sed -i.bak -E 's/(createTransfer\(.*\)):.*$/\1: Promise<TransactionData>;/g' $PARENT_DIR/$1/libzkbob_rs_wasm.d.ts
-  sed -i.bak -E 's/(createTransferOptimistic\(.*\)):.*$/\1: Promise<TransactionData>;/g' $PARENT_DIR/$1/libzkbob_rs_wasm.d.ts
-  sed -i.bak -E 's/(createWithdraw\(.*\)):.*$/\1: Promise<TransactionData>;/g' $PARENT_DIR/$1/libzkbob_rs_wasm.d.ts
-  sed -i.bak -E 's/(createWithdrawalOptimistic\(.*\)):.*$/\1: Promise<TransactionData>;/g' $PARENT_DIR/$1/libzkbob_rs_wasm.d.ts
+  if [ "$4" = debug ] ; then
+    wasm-opt -o $PARENT_DIR/$1/wasm-opt.wasm --debuginfo -O3 $PARENT_DIR/$1/libzkbob_rs_wasm_bg.wasm && mv $PARENT_DIR/$1/wasm-opt.wasm $PARENT_DIR/$1/libzkbob_rs_wasm_bg.wasm
+  else
+    wasm-opt -o $PARENT_DIR/$1/wasm-opt.wasm -O3 $PARENT_DIR/$1/libzkbob_rs_wasm_bg.wasm && mv $PARENT_DIR/$1/wasm-opt.wasm $PARENT_DIR/$1/libzkbob_rs_wasm_bg.wasm
+  fi
+
   # Remove invalid typings
   sed -i.bak -E '/BroccoliDestroyInstance/d' $PARENT_DIR/$1/libzkbob_rs_wasm.d.ts
   sed -i.bak -E '/BroccoliDestroyInstance/d' $PARENT_DIR/$1/libzkbob_rs_wasm_bg.wasm.d.ts
-
   rm $PARENT_DIR/$1/libzkbob_rs_wasm_bg.wasm.d.ts
 
   # Patch package.json, since wasm-pack doesn't allow renaming packages
-  sed -i.bak -E "s/\"name\": \"libzkbob-rs-wasm\"/\"name\": \"libzkbob-rs-wasm-$1\"/g" $PARENT_DIR/$1/package.json
-  # Add workerHelpers.js in package.json
-  pushd $PARENT_DIR/$1 > /dev/null
-    echo "`jq --arg WORKER_JS $(find . -name "workerHelpers.js" | sed -e "s/^.\///") '.files[.files| length] |= . + $WORKER_JS' package.json`" > package.json
-  popd > /dev/null
+  patch_package_json $1
 }
 
-build web
-#build bundler
-#build nodejs
+build web web false
+build web-mt web,multicore true
+build web-debug web false debug
+build web-mt-debug web,multicore true debug

libzkbob-rs-wasm/scripts/clean

@@ -2,4 +2,4 @@
 
 PARENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd )"
 
-rm -rf $PARENT_DIR/bundler $PARENT_DIR/nodejs $PARENT_DIR/web
+rm -rf $PARENT_DIR/web $PARENT_DIR/web-mt $PARENT_DIR/web-debug $PARENT_DIR/web-mt-debug

libzkbob-rs-wasm/scripts/publish

@@ -3,5 +3,6 @@
 PARENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd )"
 
 npm publish $PARENT_DIR/web
-#npm publish $PARENT_DIR/bundler
-#npm publish $PARENT_DIR/nodejs
+npm publish $PARENT_DIR/web-mt
+npm publish $PARENT_DIR/web-debug
+npm publish $PARENT_DIR/web-mt-debug

libzkbob-rs-wasm/src/client/coldstorage.rs

@@ -0,0 +1,20 @@
+use serde::{Deserialize, Serialize};
+use libzeropool::fawkes_crypto::ff_uint::Num;
+use crate::Fr;
+
+#[derive(Serialize, Deserialize, PartialEq, Debug)]
+pub struct TxInputData {
+    pub index: u64,
+    pub memo: Vec<u8>,
+    pub commitment: Vec<u8>,
+    pub tx_hash: Vec<u8>,
+}
+
+#[derive(Serialize, Deserialize, PartialEq, Debug)]
+pub struct BulkData {
+    pub bulk_version: u8,
+    pub index_from: u64,
+    pub root_before: Num<Fr>,
+    pub root_after: Num<Fr>,
+    pub txs: Vec<TxInputData>,
+}

libzkbob-rs-wasm/src/client/mod.rs

@@ -2,6 +2,9 @@ use std::collections::HashMap;
 use std::rc::Rc;
 use std::{cell::RefCell, convert::TryInto};
 
+#[cfg(feature = "multicore")]
+use rayon::prelude::*;
+
 use js_sys::{Array, Promise};
 use libzeropool::{
     constants,
@@ -22,25 +25,30 @@ use libzkbob_rs::{
     merkle::{Hash, Node}
 };
 use serde::{Serialize};
-use wasm_bindgen::{prelude::*, JsCast};
+use wasm_bindgen::{prelude::*, JsCast };
 use wasm_bindgen_futures::future_to_promise;
 
+use crate::ParseTxsColdStorageResult;
 use crate::client::tx_parser::StateUpdate;
 
 use crate::database::Database;
+use crate::helpers::vec_into_iter;
 use crate::ts_types::Hash as JsHash;
 use crate::{
     keys::reduce_sk, Account, Fr, Fs, Hashes, 
     IDepositData, IDepositPermittableData, ITransferData, IWithdrawData,
     IndexedNote, IndexedNotes, PoolParams, Transaction, UserState, POOL_PARAMS,
     MerkleProof, Pair, TreeNode, TreeNodes,
 };
-
-mod tx_types;
 use tx_types::JsTxType;
+use crate::client::coldstorage::{ BulkData };
+use crate::client::tx_parser::{ ParseResult, ParseColdStorageResult };
 
+mod tx_types;
+mod coldstorage;
 mod tx_parser;
 
+
 // TODO: Find a way to expose MerkleTree,
 
 #[wasm_bindgen]
@@ -307,6 +315,10 @@ impl UserAccount {
             None => None
         };
 
+        Ok(self.update_state_internal(state_update))
+    }
+
+    fn update_state_internal(&mut self, state_update: StateUpdate) -> () {
         if !state_update.new_leafs.is_empty() || !state_update.new_commitments.is_empty() {
             self.inner.borrow_mut()
                 .state
@@ -328,7 +340,96 @@ impl UserAccount {
             });
         });
 
-        Ok(())
+        ()
+    }
+
+
+    #[wasm_bindgen(js_name = "updateStateColdStorage")]
+    pub fn update_state_cold_storage(
+        &mut self,
+        bulks: Vec<js_sys::Uint8Array>,
+        from_index: Option<u64>,    // inclusively
+        to_index: Option<u64>,      // exclusively
+    ) -> Result<ParseTxsColdStorageResult, JsValue> {
+        const MAX_SUPPORTED_BULK_VERSION: u8 = 1;
+
+        let mut total_txs_cnt: usize = 0;
+        let bulks_obj: Result<Vec<BulkData>, JsValue> = bulks.into_iter().map(|array| {
+            let bulk_data = array.to_vec();
+            let bulk: BulkData = match bincode::deserialize(&bulk_data) {
+                Ok(res) => res,
+                Err(e) => return Err(js_err!(&format!("Cannot parse bulk data: {}", e))),
+            };
+
+            if bulk.bulk_version > MAX_SUPPORTED_BULK_VERSION {
+                return Err(js_err!(&format!("Incorrect bluk vesion {}, supported {}", bulk.bulk_version, MAX_SUPPORTED_BULK_VERSION)))
+            }
+
+            Ok(bulk)
+        })
+        .collect();
+
+        if let Err(e) = bulks_obj {
+            return Err(e);
+        }
+
+
+        let mut single_result: ParseResult = bulks_obj
+            .unwrap()
+            .into_iter()
+            .map(|bulk| -> Vec<ParseResult> {
+                let eta = &self.inner.borrow().keys.eta;
+                let params = &self.inner.borrow().params;
+                let range = from_index.unwrap_or(0)..to_index.unwrap_or(u64::MAX);
+                let bulk_results: Vec<ParseResult> = vec_into_iter(bulk.txs)
+                    .filter(|tx| range.contains(&tx.index))
+                    .map(|tx| -> ParseResult {
+                        tx_parser::parse_tx(
+                            tx.index,
+                            &tx.commitment,
+                            &tx.memo,
+                            Some(&tx.tx_hash),
+                            eta,
+                            params
+                        )
+                    })
+                    .collect();
+
+                total_txs_cnt = total_txs_cnt + bulk_results.len();
+
+                bulk_results
+            })
+            .flatten()
+            .fold(Default::default(), |acc: ParseResult, parse_result| {
+                ParseResult {
+                    decrypted_memos: vec![acc.decrypted_memos, parse_result.decrypted_memos].concat(),
+                    state_update: StateUpdate {
+                        new_leafs: vec![acc.state_update.new_leafs, parse_result.state_update.new_leafs].concat(),
+                        new_commitments: vec![acc.state_update.new_commitments, parse_result.state_update.new_commitments].concat(),
+                        new_accounts: vec![acc.state_update.new_accounts, parse_result.state_update.new_accounts].concat(),
+                        new_notes: vec![acc.state_update.new_notes, parse_result.state_update.new_notes].concat()
+                    }
+                }
+            });
+
+        let decrypted_leafs_cnt: usize = single_result.state_update.new_leafs.len();
+
+        self.update_state_internal(single_result.state_update);
+
+
+        single_result.decrypted_memos.sort_by(|a,b| a.index.cmp(&b.index));
+
+        let sync_result = ParseColdStorageResult {
+            decrypted_memos: single_result.decrypted_memos,
+            tx_cnt: total_txs_cnt,
+            decrypted_leafs_cnt: decrypted_leafs_cnt,
+        };
+
+        let sync_result = serde_wasm_bindgen::to_value(&sync_result)
+            .unwrap()
+            .unchecked_into::<ParseTxsColdStorageResult>();
+
+        Ok(sync_result)
     }
 
     #[wasm_bindgen(js_name = "getRoot")]