Switch to new kvdb-rocksdb (paritytech#4186)

* Switch to new rocksdb with some hacks to resolve linking errors Rocksdb and `wasmtime-environ` both link `zstd`. In the final link step, this leads to linking errors because of duplicate symbols. The linked Rocksdb fixes this by using `zstd-sys` as well. However, this currently also requires modifications in `zstd-sys`. Someone will need to come up with a better implementation. * Switch to new version of kvdb-rocksdb * Update client/db/src/utils.rs Co-Authored-By: Shawn Tabrizi <shawntabrizi@gmail.com>
hicommonwealth · Nov 23, 2019 · eef00ea · eef00ea
1 parent c9e964f
commit eef00ea
Show file tree

Hide file tree

Showing 8 changed files with 89 additions and 88 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml
@@ -92,7 +92,7 @@ console_log = { version = "0.1.2", optional = true }
 js-sys = { version = "0.3.22", optional = true }
 wasm-bindgen = { version = "0.2.45", optional = true }
 wasm-bindgen-futures = { version = "0.3.22", optional = true }
-kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d", optional = true }
+kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="03a2ba08f47f4af4219280e660a1ea92cb8896bd", optional = true }
 rand6 = { package = "rand", version = "0.6", features = ["wasm-bindgen"], optional = true }	# Imported just for the `wasm-bindgen` feature
 
 [dev-dependencies]

diff --git a/client/Cargo.toml b/client/Cargo.toml
@@ -19,7 +19,7 @@ header-metadata = { package = "substrate-header-metadata", path = "header-metada
 hex-literal = { version = "0.2.1" }
 inherents = { package = "substrate-inherents", path = "../primitives/inherents" }
 keyring = { package = "substrate-keyring", path = "../primitives/keyring" }
-kvdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
+kvdb = { git = "https://github.com/paritytech/parity-common", rev="03a2ba08f47f4af4219280e660a1ea92cb8896bd" }
 log = { version = "0.4.8" }
 parking_lot = { version = "0.9.0" }
 primitives = { package = "substrate-primitives", path = "../primitives/core" }
@@ -36,5 +36,5 @@ env_logger = "0.7.0"
 tempfile = "3.1.0"
 client-db = { package = "substrate-client-db", path = "./db", features = ["kvdb-rocksdb"] }
 test-client = { package = "substrate-test-runtime-client", path = "../test/utils/runtime/client" }
-kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
+kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="03a2ba08f47f4af4219280e660a1ea92cb8896bd" }
 panic-handler = { package = "substrate-panic-handler", path = "../primitives/panic-handler" }
diff --git a/client/api/Cargo.toml b/client/api/Cargo.toml
@@ -18,7 +18,7 @@ header-metadata = { package = "substrate-header-metadata", path = "../header-met
 hex-literal = { version = "0.2.1" }
 inherents = { package = "substrate-inherents", path = "../../primitives/inherents", default-features = false }
 keyring = { package = "substrate-keyring", path = "../../primitives/keyring" }
-kvdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
+kvdb = { git = "https://github.com/paritytech/parity-common", rev="03a2ba08f47f4af4219280e660a1ea92cb8896bd" }
 log = { version = "0.4.8" }
 parking_lot = { version = "0.9.0" }
 primitives = { package = "substrate-primitives", path = "../../primitives/core", default-features = false }
@@ -36,6 +36,6 @@ tempfile = "3.1.0"
 client-db = { package = "substrate-client-db", path = ".././db", features = ["kvdb-rocksdb"] }
 test-primitives = { package = "substrate-test-primitives", path = "../../test/utils/primitives" }
 test-client = { package = "substrate-test-runtime-client", path = "../../test/utils/runtime/client" }
-kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
+kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="03a2ba08f47f4af4219280e660a1ea92cb8896bd" }
 panic-handler = { package = "substrate-panic-handler", path = "../../primitives/panic-handler" }
 
diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml
@@ -7,10 +7,10 @@ edition = "2018"
 [dependencies]
 parking_lot = "0.9.0"
 log = "0.4.8"
-kvdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
+kvdb = { git = "https://github.com/paritytech/parity-common", rev="03a2ba08f47f4af4219280e660a1ea92cb8896bd" }
 # FIXME replace with release as soon as our rocksdb changes are released upstream https://github.com/paritytech/parity-common/issues/88
-kvdb-rocksdb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d", optional = true }
-kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="b0317f649ab2c665b7987b8475878fc4d2e1f81d" }
+kvdb-rocksdb = { git = "https://github.com/paritytech/parity-common", rev="03a2ba08f47f4af4219280e660a1ea92cb8896bd", optional = true }
+kvdb-memorydb = { git = "https://github.com/paritytech/parity-common", rev="03a2ba08f47f4af4219280e660a1ea92cb8896bd" }
 linked-hash-map = "0.5.2"
 hash-db = "0.15.2"
 client-api = { package = "substrate-client-api", path = "../api" }

diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs
@@ -98,7 +98,11 @@ pub struct RefTrackingState<Block: BlockT> {
 }
 
 impl<B: BlockT> RefTrackingState<B> {
-	fn new(state: DbState, storage: Arc<StorageDb<B>>, parent_hash: Option<B::Hash>) -> RefTrackingState<B> {
+	fn new(
+		state: DbState,
+		storage: Arc<StorageDb<B>>,
+		parent_hash: Option<B::Hash>,
+	) -> RefTrackingState<B> {
 		RefTrackingState {
 			state,
 			parent_hash,

diff --git a/client/db/src/offchain.rs b/client/db/src/offchain.rs
@@ -43,7 +43,7 @@ impl LocalStorage {
 	/// Create new offchain storage for tests (backed by memorydb)
 	#[cfg(any(test, feature = "test-helpers"))]
 	pub fn new_test() -> Self {
-		let db = Arc::new(::kvdb_memorydb::create(crate::utils::NUM_COLUMNS));
+		let db = Arc::new(kvdb_memorydb::create(crate::utils::NUM_COLUMNS));
 		Self::new(db as _)
 	}
 

diff --git a/client/db/src/utils.rs b/client/db/src/utils.rs
@@ -213,8 +213,24 @@ pub fn open_database(
 		#[cfg(feature = "kvdb-rocksdb")]
 		DatabaseSettingsSrc::Path { path, cache_size } => {
 			let mut db_config = DatabaseConfig::with_columns(Some(NUM_COLUMNS));
-			db_config.memory_budget = *cache_size;
-			let path = path.to_str().ok_or_else(|| client::error::Error::Backend("Invalid database path".into()))?;
+
+			if let Some(cache_size) = cache_size {
+				let state_col_budget = (*cache_size as f64 * 0.7) as usize;
+				let other_col_budget = cache_size - state_col_budget;
+
+				let mut memory_budget = std::collections::HashMap::new();
+				for i in 0..NUM_COLUMNS {
+					if Some(i) == crate::columns::STATE {
+						memory_budget.insert(Some(i), state_col_budget);
+					} else {
+						memory_budget.insert(Some(i), other_col_budget);
+					}
+				}
+
+				db_config.memory_budget = memory_budget;
+			}
+			let path = path.to_str()
+				.ok_or_else(|| client::error::Error::Backend("Invalid database path".into()))?;
 			Arc::new(Database::open(&db_config, &path).map_err(db_err)?)
 		},
 		#[cfg(not(feature = "kvdb-rocksdb"))]