Support typed data (#343)

* fix: replace tx with mutation

* feat: add typed data verify

* feat: support typed data

* fix: remove install open jdk

* fix: fix warning

* fix: move doc check to test

* fix: update case

* fix: add some log

* feat: meta mask test passed

.github/workflows/ci.yml

@@ -29,30 +29,6 @@ jobs:
           command: fmt
           args: --all -- --check
 
-  docs:
-    name: Docs
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-        with:
-          submodules: recursive
-      - name: Install Rust toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          profile: minimal
-          override: true
-          components: rustfmt
-      - uses: Swatinem/rust-cache@v2
-      - name: Check documentation
-        env:
-          RUSTDOCFLAGS: -D warnings
-        uses: actions-rs/cargo@v1
-        with:
-          command: doc
-          args: --no-deps --document-private-items --all-features --workspace --examples
-
   coverage:
     name: test
     runs-on: ubuntu-latest
@@ -76,9 +52,16 @@ jobs:
           cd bridge
           yarn install
           npx hardhat test
+      - name: Check documentation
+        env:
+          RUSTDOCFLAGS: -D warnings
+        uses: actions-rs/cargo@v1
+        with:
+          command: doc
+          args: --no-deps --document-private-items --all-features --workspace --examples
       - name: Setup test env
         run: |
-          sudo apt-get install protobuf-compiler openjdk-8-jdk maven
+          sudo apt-get install protobuf-compiler 
           protoc --version
           cargo build
           cd tools

Cargo.toml

@@ -15,5 +15,5 @@ members = [
 ]
 
 [workspace.dependencies]
-fastcrypto = {git = "https://github.com/MystenLabs/fastcrypto", rev = "bbb2d02a7a64c27314721748cc4d015b00490dbe"}
-
+fastcrypto = {git = "https://github.com/MystenLabs/fastcrypto", rev = "306465d4fe04f6c26359d885f3b0a548b661de40"}
+ethers = {git="https://github.com/gakonst/ethers-rs", rev="ed47eaadad7d751cff9c0f59517b0fc3fb284cd3"}

src/base/src/bson_util.rs

@@ -140,7 +140,7 @@ pub fn filter_from_json_value(json_str: &str) -> std::result::Result<Option<Filt
         let filter_doc = json_str_to_bson_document(json_str)
             .map_err(|e| DB3Error::InvalidFilterValue(format!("{:?}", e)))?;
         let field = filter_doc.get_str("field").map_err(|e| {
-            DB3Error::InvalidFilterJson("filed is required in filter json".to_string())
+            DB3Error::InvalidFilterJson(format!("filed is required in filter json for {e}"))
         })?;
         let value = match filter_doc.get("value") {
             Some(v) => filter_value_from_bson_value(v)?,
@@ -203,9 +203,7 @@ mod tests {
         bson_document_into_bytes, bson_into_comparison_bytes, bytes_to_bson_document,
         json_str_to_bson_document,
     };
-    use bson::raw::RawBson;
     use bson::Bson;
-    use bson::Document;
     use chrono::Utc;
     #[test]
     fn json_str_to_bson_document_ut() {

src/base/src/test_base.rs

@@ -21,8 +21,10 @@ use ethereum_types::Address as AccountAddress;
 use hex::FromHex;
 use rand::Rng;
 
+//
 // this function is used for testing
 //
+#[allow(dead_code)]
 pub fn get_a_static_keypair() -> Keypair {
     let secret_key: &[u8] = b"833fe62409237b9d62ec77587520911e9a759cec1d19755b7da901b96dca3d42";
     let public_key: &[u8] = b"ec172b93ad5e563bf4932c70e1245034c35467ef2efd4d64ebf819683467e2bf";
@@ -43,6 +45,7 @@ pub fn get_a_static_address() -> AccountAddress {
     get_address_from_pk(&kp.public)
 }
 
+#[allow(dead_code)]
 pub fn get_a_ts_static_keypair() -> Keypair {
     let secret_key: &[u8] = b"ea82176302fbf6b10a6c7ff25dc77b4b7dee0126841af0fc3621d7ed0ac7c9c99806d5ba5c35c68ff63850fb3f4c5dfc79135c3c2c76a560eeaee6f2135830d6";
     let public_key: &[u8] = b"9806d5ba5c35c68ff63850fb3f4c5dfc79135c3c2c76a560eeaee6f2135830d6";

src/bridge/Cargo.toml

@@ -10,7 +10,7 @@ keywords = ["database", "web3", "db3"]
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ethers = { version = "1.0.0", features = ["ws"] }
+ethers = {workspace=true, features=["ws"]}
 eyre = "0.6"
 serde = { version = "1.0.144", features = ["derive"] }
 serde_json = "1.0.64"

src/bridge/src/evm_chain_watcher.rs

@@ -22,15 +22,12 @@ use ethers::abi::RawLog;
 use ethers::types::Address;
 use ethers::types::Filter;
 use ethers::{
-    contract::{abigen, Contract, EthEvent},
-    core::types::{
-        transaction::eip2718::TypedTransaction, Log, Signature, Transaction, ValueOrArray,
-    },
+    contract::{abigen, EthEvent},
+    core::types::{transaction::eip2718::TypedTransaction, Log, Signature, Transaction},
     providers::{Middleware, Provider, StreamExt, Ws},
 };
 
 use redb::Database;
-use std::path::Path;
 use std::sync::atomic::AtomicBool;
 use std::sync::mpsc::SyncSender;
 use std::sync::Arc;

src/bridge/src/storage_chain_minter.rs

@@ -24,7 +24,7 @@ use db3_proto::db3_mutation_proto::MintCreditsMutation;
 use db3_sdk::mutation_sdk::MutationSDK;
 use db3_storage::event_store::EventStore;
 use elliptic_curve::{consts::U32, sec1::ToEncodedPoint};
-use ethers::types::{RecoveryMessage, Signature, H256, U256};
+use ethers::types::{H256, U256};
 use generic_array::GenericArray;
 use hex;
 use redb::Database;
@@ -36,7 +36,7 @@ use tracing::{info, warn};
 use k256::{
     ecdsa::{
         recoverable::{Id as RecoveryId, Signature as RecoverableSignature},
-        Error as K256SignatureError, Signature as K256Signature,
+        Signature as K256Signature,
     },
     PublicKey as K256PublicKey,
 };

src/cmd/Cargo.toml

@@ -37,7 +37,7 @@ serde = { version = "1.0.144", features = ["derive"] }
 serde_json = "1.0.88"
 hex = "0.4.3"
 fastcrypto = { workspace = true, features = ["copy_key"] }
-ethers = { version = "1.0.0", features = ["ws"] }
+ethers = {workspace=true}
 tonic = "0.8.3"
 http = "0.2"
 eyre = "0.6"

src/cmd/src/command.rs

@@ -39,7 +39,7 @@ use http::Uri;
 use prettytable::{format, Table};
 use std::sync::Arc;
 use std::time::{SystemTime, UNIX_EPOCH};
-use tonic::transport::{ClientTlsConfig, Endpoint, Server};
+use tonic::transport::{ClientTlsConfig, Endpoint};
 
 pub struct DB3ClientContext {
     pub mutation_sdk: Option<MutationSDK>,
@@ -210,7 +210,7 @@ impl DB3ClientCommand {
     fn show_document(documents: Vec<Document>) -> std::result::Result<Table, String> {
         let mut table = Table::new();
         table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
-        table.set_titles(row!["id_base64", "owner", "document", "tx_id"]);
+        table.set_titles(row!["id_base64", "owner", "document", "mutation_id"]);
         let mut error_cnt = 0;
         for document in documents {
             if let Ok(id) = DocumentId::try_from_bytes(document.id.as_slice()) {
@@ -263,7 +263,7 @@ impl DB3ClientCommand {
         table.set_titles(row![
             "database address",
             "sender address",
-            "related transactions",
+            "related mutations",
             "collections"
         ]);
         let tx_list: String = database
@@ -510,7 +510,7 @@ impl DB3ClientCommand {
                         println!("send add collection done!");
                         let mut table = Table::new();
                         table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
-                        table.set_titles(row!["tx_id"]);
+                        table.set_titles(row!["mutation_id"]);
                         table.add_row(row![tx_id.to_base64()]);
                         Ok(table)
                     }
@@ -615,7 +615,7 @@ impl DB3ClientCommand {
                     Ok((db_id, tx_id)) => {
                         let mut table = Table::new();
                         table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
-                        table.set_titles(row!["database address", "transaction id"]);
+                        table.set_titles(row!["database address", "mutation id"]);
                         table.add_row(row![db_id.to_hex(), tx_id.to_base64()]);
                         Ok(table)
                     }
@@ -666,7 +666,7 @@ impl DB3ClientCommand {
                         println!("send add document done");
                         let mut table = Table::new();
                         table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
-                        table.set_titles(row!["transaction id"]);
+                        table.set_titles(row!["mutation id"]);
                         table.add_row(row![tx_id.to_base64()]);
                         Ok(table)
                     }
@@ -722,10 +722,9 @@ impl DB3ClientCommand {
                     .await
                 {
                     Ok((_, tx_id)) => {
-                        println!("send update document done");
                         let mut table = Table::new();
                         table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
-                        table.set_titles(row!["transaction id"]);
+                        table.set_titles(row!["mutation id"]);
                         table.add_row(row![tx_id.to_base64()]);
                         Ok(table)
                     }
@@ -774,7 +773,7 @@ impl DB3ClientCommand {
                         println!("send delete document done");
                         let mut table = Table::new();
                         table.set_format(*format::consts::FORMAT_NO_BORDER_LINE_SEPARATOR);
-                        table.set_titles(row!["transaction id"]);
+                        table.set_titles(row!["mutation id"]);
                         table.add_row(row![tx_id.to_base64()]);
                         Ok(table)
                     }

src/cmd/src/deposit.rs

@@ -15,13 +15,12 @@
 // limitations under the License.
 //
 
-use ethers::providers::Middleware;
 use ethers::{
     contract::abigen,
     core::types::{Address, U256},
     middleware::SignerMiddleware,
     providers::{Provider, Ws},
-    signers::{LocalWallet, Signer},
+    signers::LocalWallet,
 };
 
 use eyre::Result;
@@ -51,16 +50,15 @@ pub async fn lock_balance(
     let provider_arc = Arc::new(provider);
     let token_address = erc20_token_addr.parse::<Address>().unwrap();
     let rollup_address = rollup_addr.parse::<Address>().unwrap();
-    let my_address = wallet.address();
     let signable_client = SignerMiddleware::new(provider_arc.clone(), wallet);
     let client = Arc::new(signable_client);
     let token_contract = DB3TokenContract::new(token_address, client.clone());
     let approve_amount = U256::from(100_000_000_000 as u64); // 10 db3
     let approve_request = token_contract.approve(rollup_address, approve_amount);
-    let result = approve_request.send().await;
+    let _result = approve_request.send().await;
     let rollup_contract = DB3RollupContract::new(rollup_address, client);
     let deposit_amount = U256::from((amount * 1000_000_000.0) as u64);
     let deposit_request = rollup_contract.deposit(deposit_amount);
-    let result = deposit_request.send().await;
+    let _result = deposit_request.send().await;
     Ok(())
 }

src/cmd/src/show_evm_account.rs

@@ -17,9 +17,8 @@
 
 use ethers::{
     contract::abigen,
-    core::types::{Address, TransactionRequest, U256},
+    core::types::{Address, U256},
     providers::{Provider, Ws},
-    signers::{LocalWallet, Signer},
 };
 
 use eyre::Result;

src/crypto/Cargo.toml

@@ -18,15 +18,17 @@ anyhow = "1.0.68"
 prost = "0.11"
 prost-types = "0.11"
 rand = "0.8.5"
-bytes = "1"
+#bytes = "1"
+bytes = { version = "1.3.0", features = ["serde"] }
 hex = "0.4.3"
 base64ct = { version = "1.5.3", features = ["alloc"] }
 schemars ="0.8.10"
 serde = { version = "1.0.144", features = ["derive"] }
 serde-name = "0.2.1"
 thiserror = "1.0.34"
-serde_bytes = "0.11.7"
-serde_json = "1.0.88"
+#serde_bytes = "0.11.7"
+#serde_json = "1.0.88"
+serde_json = { version = "1.0.64", default-features = false }
 serde_with = "2.1.0"
 serde_repr = "0.1"
 signature = "1.6.0"
@@ -38,7 +40,7 @@ slip10_ed25519 = "0.1.3"
 byteorder = "1.4.3"
 rust_secp256k1 = { version = "0.24.0", package = "secp256k1", features = ["bitcoin_hashes"] }
 bip32 = "0.4.0"
-
+ethers = { workspace = true }
 [dev-dependencies]
 tiny-bip39 = "1.0.0"
 

src/crypto/src/db3_keypair.rs

@@ -52,6 +52,14 @@ impl DB3KeyPair {
             DB3KeyPair::Secp256k1(kp) => DB3PublicKey::Secp256k1(kp.public().clone()),
         }
     }
+    pub fn try_sign_hashed_message(&self, msg: &[u8]) -> std::result::Result<Vec<u8>, DB3Error> {
+        match self {
+            DB3KeyPair::Ed25519(_) => Err(DB3Error::SignError(
+                "signing hashed message is not supperted with ed25519".to_string(),
+            )),
+            DB3KeyPair::Secp256k1(kp) => Secp256k1DB3Signature::new_hashed(&kp, msg),
+        }
+    }
 }
 
 impl Signer<Signature> for DB3KeyPair {
@@ -167,6 +175,7 @@ mod tests {
     use crate::db3_signature::DB3Signature;
     use crate::key_derive;
     use bip39::{Language, Mnemonic, Seed};
+    use fastcrypto::hash::{HashFunction, Sha3_256};
     use hex;
     #[test]
     fn keypair_smoke_test_secp256k1() {
@@ -256,21 +265,4 @@ mod tests {
         let is_ok = ts_signature_ret.unwrap().verify(msg.as_ref());
         assert_eq!(true, is_ok.is_ok());
     }
-
-    #[test]
-    fn test_generate_evm_address() {
-        let private_key_hex = "ad689d9b7751da07b0fb39c5091672cbfe50f59131db015f8a0e76c9790a6fcc";
-        let data = hex::decode(private_key_hex).unwrap();
-        let result = Secp256k1PrivateKey::from_bytes(data.as_ref());
-        assert!(result.is_ok());
-        let kp = Secp256k1KeyPair::from(result.unwrap());
-        let db3_kp = DB3KeyPair::Secp256k1(kp);
-        let msg: [u8; 1] = [0; 1];
-        let signature = db3_kp.try_sign(&msg).unwrap();
-        let db3_address = signature.verify(&msg).unwrap();
-        assert_eq!(
-            "\"0x94fdb38548dca0df72b2ff43ce0d77a8867dd123\"",
-            serde_json::to_string(&db3_address).unwrap()
-        );
-    }
 }

src/crypto/src/db3_signature.rs

@@ -27,6 +27,7 @@ use fastcrypto::secp256k1::{Secp256k1KeyPair, Secp256k1PublicKey, Secp256k1Signa
 use fastcrypto::traits::KeyPair as KeypairTraits;
 use fastcrypto::traits::{Authenticator, ToFromBytes, VerifyingKey};
 use fastcrypto::Verifier;
+use rust_secp256k1::{Message, Secp256k1};
 use schemars::JsonSchema;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use serde_with::{serde_as, Bytes};
@@ -169,6 +170,24 @@ pub struct Secp256k1DB3Signature(
     [u8; Secp256k1PublicKey::LENGTH + Secp256k1Signature::LENGTH + 1],
 );
 
+impl Secp256k1DB3Signature {
+    pub fn new_hashed(kp: &Secp256k1KeyPair, msg: &[u8]) -> Result<Vec<u8>> {
+        let secp = Secp256k1::signing_only();
+        let message = Message::from_slice(msg)
+            .map_err(|e| DB3Error::InvalidSignature(format!("bad message for {e}")))?;
+        let sig = secp.sign_ecdsa_recoverable(&message, &kp.secret.privkey);
+        let (recovery_id, sig) = sig.serialize_compact();
+        let mut signature_bytes: Vec<u8> =
+            Vec::with_capacity(Secp256k1PublicKey::LENGTH + Secp256k1Signature::LENGTH + 1);
+        let scheme = SignatureScheme::Secp256k1;
+        signature_bytes.extend_from_slice(&[scheme.flag()]);
+        signature_bytes.extend_from_slice(&sig);
+        signature_bytes.extend_from_slice(&[recovery_id.to_i32() as u8]);
+        signature_bytes.extend_from_slice(kp.public().as_ref());
+        Ok(signature_bytes)
+    }
+}
+
 impl AsRef<[u8]> for Secp256k1DB3Signature {
     fn as_ref(&self) -> &[u8] {
         self.0.as_ref()