feat: add storage

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "thirdparty/msql-srv"]
 	path = thirdparty/msql-srv
 	url = https://github.com/imotai/msql-srv.git
+[submodule "thirdparty/merk"]
+	path = thirdparty/merk
+	url = https://github.com/nomic-io/merk.git

CHANGELOG.md

@@ -3,5 +3,4 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
 ## [Unreleased]

Cargo.toml

@@ -2,6 +2,9 @@
 members = [
     "src/crypto",
     "src/proto",
-    "src/kvstore",
     "src/error",
+    "src/types",
+    "src/base",
+    "src/storage",
+    "src/app"
 ]

clippy.toml

@@ -0,0 +1,8 @@
+# cyclomatic complexity is not always useful
+cognitive-complexity-threshold = 100
+# types are used for safety encoding
+type-complexity-threshold = 10000
+# manipulating complex states machines in consensus
+too-many-arguments-threshold = 13
+# Reasonably large enum variants are okay
+enum-variant-size-threshold = 1000

docs/mutation.md

@@ -6,6 +6,7 @@ Mutation is the `transaction` of db3 and the insert and update operations will b
 
 * batch insert kv pairs
 * batch update kv pairs
+* batch delete kv pairs
 * support namespace
 
 ## The problems that Mutation will solve
@@ -37,8 +38,8 @@ enum ChainId {
 ```protobuf=
 
 enum MutationAction {
-    Insert_Action = 0; // this action will fail if the key exists
-    Update_Action = 1; // this action will fail if the key does not exist
+    Insert_Action = 0;
+    Delete_Action = 1;
 }
 
 message KVPair {
@@ -75,40 +76,6 @@ message WriteRequest {
 }
 ```
 
-## 通过签名与验证方式实现身份认证
-
-
-db3使用[fastcrypto](https://github.com/MystenLabs/fastcrypto/)库来实现身份认证，
-![sign](./images/db3-sign.png)
-
-```rust
-let kv = KvPair{
-            key:"k1".as_bytes().to_vec(),
-            value:"value1".as_bytes().to_vec(),
- };  
-let mutation = Mutation {
-    ns: "my_twitter".as_bytes().to_vec(),
-    kv_pairs:vec![kv],
-    nonce:1,
-    chain_id:ChainId::MainNet.into(),
-    chain_role:ChainRole::StorageShardChain.into(),
-};  
-let mut buf = BytesMut::with_capacity(1024 * 4); 
-mutation.encode(&mut buf);
-let buf = buf.freeze();
-let signature: Secp256k1Signature = kp.sign(buf.as_ref());
-let request = WriteRequest {
-    signature: signature.as_ref().to_vec(),
-    mutation:buf.as_ref().to_vec(),
-    public_key: kp.public().as_ref().to_vec()
-};  
-let mut buf = BytesMut::with_capacity(1024 * 4); 
-request.encode(&mut buf);
-let buf = buf.freeze();
-println!("request 0x{}",hex::encode(buf.as_ref()));
-```
-output
-```shell
-0x0a41c5286f85fd7916d1c87a69bd9fa1d7c119b1aa265891e58a05cf609535e4d7e243d18e76048494e226e890d89b516a8d91f57a3be34b30663819dc502679315600121e0a0a6d795f74776974746572120c0a026b31120676616c7565311801280a1a2102337cca2171fdbfcfd657fa59881f46269f1e590b5ffab6023686c7ad2ecc2c1c
-```
+## Sign Mutation
+![flow of sign](./images/db3_sign_mutation.png)
 

rustfmt.toml

@@ -0,0 +1,2 @@
+edition = "2021"
+use_field_init_shorthand = true

src/kvstore/Cargo.toml → src/app/Cargo.toml

@@ -4,32 +4,33 @@ version = "0.1.0"
 edition = "2021"
 
 [[bin]]
-name = "db3-kvstore"
+name = "db3"
 path = "src/main.rs"
-required-features = [ "binary"]
 [features]
 default = []
 client = []
 echo-app = []
 kvstore-app = []
-binary = [
-    "structopt",
-    "tracing-subscriber/fmt",
-]
 
 [dependencies]
 bytes = { version = "1.0", default-features = false }
 tendermint-abci = {version = "0.25.0", git = "https://github.com/dbpunk-labs/tendermint-rs.git", branch = "release/v0.25.0"}
 tendermint-proto = {version = "0.25.0", git = "https://github.com/dbpunk-labs/tendermint-rs.git", branch = "release/v0.25.0"}
 tracing = { version = "0.1", default-features = false }
 flex-error = { version = "0.4.4", default-features = false }
-structopt = { version = "0.3", optional = true, default-features = false }
+structopt = { version = "0.3" }
 db3-proto={path="../proto", version="0.1.0"}
 db3-crypto={path="../crypto", version="0.1.0"}
+db3-storage={path="../storage", version="0.1.0"}
+db3-types={path="../types", version="0.1.0"}
 prost = "0.10"
 prost-types = "0.10"
-tracing-subscriber = { version = "0.2", optional = true, default-features = false }
+tracing-subscriber = { version = "0.2"}
 fastcrypto="0.1.3"
 signature = { version = "1.6.4", features = ["rand-preview"] }
 hex = "0.4.3"
 rand = "0.8.5"
+rust_secp256k1 = { version = "0.24.0", package = "secp256k1", features = ["recovery", "rand-std", "bitcoin_hashes", "global-context"] }
+merk = {version= "2.0.0", path = "../../thirdparty/merk"}
+ethereum-types = { version = "0.13.1", default-features = false }
+tempdir = "0.3.7"

src/app/src/lib.rs

@@ -0,0 +1,228 @@
+//! In-memory key/value store ABCI application.
+
+use std::{
+    collections::HashMap,
+    sync::mpsc::{channel, Receiver, Sender},
+};
+
+use bytes::BytesMut;
+use db3_crypto::verifier;
+use db3_proto::db3_bill_proto::{Bill, BillType};
+use db3_proto::db3_mutation_proto::{Mutation, WriteRequest};
+use db3_storage::bill_store::BillStore;
+use db3_storage::kv_store::KvStore;
+use db3_types::cost;
+/// In-memory, hashmap-backed key/value store ABCI application.
+///
+/// This structure effectively just serves as a handle to the actual key/value
+/// store - the [`KeyValueStoreDriver`].
+///
+///
+use ethereum_types::Address as AccountAddress;
+use hex;
+use merk::Merk;
+use prost::Message;
+use rust_secp256k1::Message as HashMessage;
+use std::boxed::Box;
+use std::pin::Pin;
+use std::sync::{Arc, Mutex};
+use tendermint_abci::codec::MAX_VARINT_LENGTH;
+use tendermint_abci::{codec, Application, Error};
+use tendermint_proto::abci::{
+    Event, RequestBeginBlock, RequestCheckTx, RequestDeliverTx, RequestInfo, RequestQuery,
+    ResponseBeginBlock, ResponseCheckTx, ResponseCommit, ResponseDeliverTx, ResponseInfo,
+    ResponseQuery,
+};
+use tracing::{debug, info};
+
+pub struct InternalState {
+    last_block_height: i64,
+    last_block_app_hash: Vec<u8>,
+    db: Pin<Box<Merk>>,
+    pending_mutation: Vec<(AccountAddress, Mutation)>,
+    tmp_id: u64,
+    pending_bills: Vec<(AccountAddress, Bill)>,
+    current_block_height: i64,
+    current_block_app_hash: Vec<u8>,
+}
+
+impl std::fmt::Debug for InternalState {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "state height {} ", self.last_block_height)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct KeyValueStoreApp {
+    state: Arc<Mutex<Pin<Box<InternalState>>>>,
+}
+
+impl KeyValueStoreApp {
+    /// Constructor.
+    pub fn new(merk: Merk) -> Self {
+        Self {
+            state: Arc::new(Mutex::new(Box::pin(InternalState {
+                last_block_height: 0,
+                last_block_app_hash: vec![],
+                db: Box::pin(merk),
+                pending_mutation: Vec::new(),
+                tmp_id: 0,
+                pending_bills: Vec::new(),
+                current_block_height: 0,
+                current_block_app_hash: vec![],
+            }))),
+        }
+    }
+}
+
+impl Application for KeyValueStoreApp {
+    fn info(&self, request: RequestInfo) -> ResponseInfo {
+        debug!(
+            "Got info request. Tendermint version: {}; Block version: {}; P2P version: {}",
+            request.version, request.block_version, request.p2p_version
+        );
+        match self.state.lock() {
+            Ok(s) => ResponseInfo {
+                data: "db3".to_string(),
+                version: "0.1.0".to_string(),
+                app_version: 1,
+                last_block_height: s.last_block_height,
+                last_block_app_hash: s.last_block_app_hash.to_vec(),
+            },
+            Err(_) => ResponseInfo {
+                data: "db3".to_string(),
+                version: "0.1.0".to_string(),
+                app_version: 1,
+                last_block_height: 0,
+                last_block_app_hash: vec![],
+            },
+        }
+    }
+
+    fn begin_block(&self, request: RequestBeginBlock) -> ResponseBeginBlock {
+        match self.state.lock() {
+            Ok(mut s) => {
+                if let Some(header) = request.header {
+                    s.current_block_height = header.height;
+                }
+            }
+            Err(_) => todo!(),
+        }
+        Default::default()
+    }
+
+    fn query(&self, request: RequestQuery) -> ResponseQuery {
+        Default::default()
+    }
+
+    fn check_tx(&self, request: RequestCheckTx) -> ResponseCheckTx {
+        let tx = String::from_utf8(request.tx).unwrap();
+        let buf = hex::decode(tx).unwrap();
+        let request = WriteRequest::decode(buf.as_ref()).unwrap();
+        let account_id = verifier::MutationVerifier::verify(&request);
+        match account_id {
+            Ok(_) => ResponseCheckTx {
+                code: 0,
+                data: vec![],
+                log: "".to_string(),
+                info: "".to_string(),
+                gas_wanted: 1,
+                gas_used: 0,
+                events: vec![],
+                codespace: "".to_string(),
+                ..Default::default()
+            },
+            Err(_) => ResponseCheckTx {
+                code: 1,
+                data: vec![],
+                log: "".to_string(),
+                info: "".to_string(),
+                gas_wanted: 1,
+                gas_used: 0,
+                events: vec![],
+                codespace: "".to_string(),
+                ..Default::default()
+            },
+        }
+    }
+
+    fn deliver_tx(&self, request: RequestDeliverTx) -> ResponseDeliverTx {
+        let tx = String::from_utf8(request.tx).unwrap();
+        let buf = hex::decode(tx).unwrap();
+        let request = WriteRequest::decode(buf.as_ref()).unwrap();
+        let account_id = verifier::MutationVerifier::verify(&request).unwrap();
+        let mutation = Mutation::decode(request.mutation.as_ref()).unwrap();
+        let mutation_id = HashMessage::from_hashed_data::<rust_secp256k1::hashes::sha256::Hash>(
+            request.mutation.as_ref(),
+        );
+        //TODO check nonce
+        match self.state.lock() {
+            Ok(mut s) => {
+                // add mu
+                let gas_fee = cost::estimate_gas(&mutation);
+                s.pending_mutation.push((account_id.addr, mutation));
+                s.tmp_id = s.tmp_id + 1;
+                let bill = Bill {
+                    gas_fee,
+                    block_height: s.current_block_height as u64,
+                    bill_id: s.tmp_id,
+                    bill_type: BillType::BillForMutation.into(),
+                    time: 0,
+                    bill_target_id: mutation_id.as_ref().to_vec(),
+                };
+                s.pending_bills.push((account_id.addr, bill));
+            }
+            Err(_) => {}
+        }
+        ResponseDeliverTx {
+            code: 0,
+            data: vec![],
+            log: "".to_string(),
+            info: "".to_string(),
+            gas_wanted: 0,
+            gas_used: 0,
+            events: vec![Event {
+                r#type: "app".to_string(),
+                attributes: vec![],
+            }],
+            codespace: "".to_string(),
+        }
+    }
+
+    fn commit(&self) -> ResponseCommit {
+        match self.state.lock() {
+            Ok(mut s) => {
+                let mutations = &s.pending_mutation.to_vec();
+                let bills = &s.pending_bills.to_vec();
+                for (addr, mutation) in mutations {
+                    let db: Pin<&mut Merk> = Pin::as_mut(&mut s.db);
+                    KvStore::apply(db, &addr, &mutation).unwrap();
+                }
+                for (addr, bill) in bills {
+                    let db: Pin<&mut Merk> = Pin::as_mut(&mut s.db);
+                    BillStore::apply(db, &addr, &bill).unwrap();
+                }
+                s.pending_mutation.clear();
+                s.pending_bills.clear();
+                s.last_block_app_hash = s.db.root_hash().to_vec();
+                s.current_block_app_hash = vec![];
+                s.last_block_height = s.current_block_height;
+                s.current_block_height = 0;
+                ResponseCommit {
+                    data: s.last_block_app_hash.to_vec(),
+                    retain_height: s.last_block_height,
+                }
+            }
+            Err(_) => {
+                // never go to here
+                ResponseCommit {
+                    data: vec![],
+                    retain_height: 0,
+                }
+            }
+        }
+    }
+}
+unsafe impl Send for KeyValueStoreApp {}
+
+unsafe impl Sync for KeyValueStoreApp {}