diff --git a/tool/state-processor/Cargo.lock b/tool/state-processor/Cargo.lock index ba158446d..8365b0cd8 100644 --- a/tool/state-processor/Cargo.lock +++ b/tool/state-processor/Cargo.lock @@ -23,6 +23,12 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a913633b0c922e6b745072795f50d90ebea78ba31a57e2ac8c2fc7b50950949" +[[package]] +name = "array-bytes" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f72e9d6fac4bc80778ea470b20197b88d28c292bb7d60c3fb099280003cd19" + [[package]] name = "arrayvec" version = "0.4.12" @@ -439,7 +445,7 @@ name = "state-processor" version = "0.0.0" dependencies = [ "anyhow", - "array-bytes", + "array-bytes 6.0.0", "fxhash", "log", "parity-scale-codec", @@ -484,7 +490,7 @@ name = "substorager" version = "0.9.0-rc16" source = "git+https://github.com/hack-ink/subalfred#de1ed2b16b0b2c3ceeb61b1f9168401bc0ace51d" dependencies = [ - "array-bytes", + "array-bytes 4.1.0", "subhasher", ] diff --git a/tool/state-processor/Cargo.toml b/tool/state-processor/Cargo.toml index 829a5f9e8..eb70123db 100644 --- a/tool/state-processor/Cargo.toml +++ b/tool/state-processor/Cargo.toml @@ -1,20 +1,20 @@ [package] authors = ["Xavier Lau "] edition = "2021" -license = "GPL-3.0" name = "state-processor" version = "0.0.0" [dependencies] # crates.io anyhow = { version = "1.0" } -array-bytes = { version = "4.1" } +array-bytes = { version = "6.0" } fxhash = { version = "0.2" } log = { version = "0.4" } parity-scale-codec = { version = "3.2", features = ["derive"] } pretty_env_logger = { version = "0.4" } serde = { version = "1.0" } serde_json = { version = "1.0" } + # hack-ink subhasher = { git = "https://github.com/hack-ink/subalfred" } subspector = { git = "https://github.com/hack-ink/subalfred" } diff --git a/tool/state-processor/src/balances/mod.rs b/tool/state-processor/src/balances/mod.rs index ec9d4dbc5..be52f97fe 100644 --- a/tool/state-processor/src/balances/mod.rs +++ b/tool/state-processor/src/balances/mod.rs @@ -1,37 +1,89 @@ // darwinia use crate::*; +type Locks = Vec; + impl Processor { - pub fn process_balances( - &mut self, - ring_locks: &mut Map>, - kton_locks: &mut Map>, - ) -> &mut Self { - log::info!("take solo balance locks"); + pub fn process_balances(&mut self) -> (u128, u128) { + let mut solo_ring_total_issuance = u128::default(); + let mut kton_total_issuance = u128::default(); + let mut solo_ring_locks = >::default(); + let mut solo_kton_locks = >::default(); + let mut para_ring_locks = >::default(); + let mut para_ring_total_issuance = u128::default(); + + log::info!("take solo balances total issuances and locks"); self.solo_state - .take::, _>( - b"Balances", - b"Locks", - ring_locks, - get_blake2_128_concat_suffix, - ) - .take::, _>( - b"Kton", - b"Locks", - kton_locks, - get_blake2_128_concat_suffix, - ); - - // --- - // Currently, there are only fee-market locks. - // I suggest shutting down the fee-market before merging. - // So, we could ignore the para balance locks migration. - // --- - - log::info!("adjust solo balance lock decimals"); - ring_locks.iter_mut().for_each(|(_, v)| v.iter_mut().for_each(|l| l.amount *= GWEI)); - kton_locks.iter_mut().for_each(|(_, v)| v.iter_mut().for_each(|l| l.amount *= GWEI)); - - self + .take_value(b"Balances", b"TotalIssuance", &mut solo_ring_total_issuance) + .take_value(b"Kton", b"TotalIssuance", &mut kton_total_issuance) + .take_map(b"Balances", b"Locks", &mut solo_ring_locks, get_hashed_key) + .take_map(b"Kton", b"Locks", &mut solo_kton_locks, get_hashed_key); + + log::info!("prune solo balance locks"); + prune(&mut solo_ring_locks); + prune(&mut solo_kton_locks); + + log::info!("adjust solo balances items' decimals"); + solo_ring_total_issuance *= GWEI; + kton_total_issuance *= GWEI; + // solo_ring_locks.iter_mut().for_each(|(_, v)| v.iter_mut().for_each(|l| l.amount *= + // GWEI)); solo_kton_locks.iter_mut().for_each(|(_, v)| v.iter_mut().for_each(|l| l.amount + // *= GWEI)); + + log::info!("take para balances total issuances and locks"); + self.para_state + .take_value(b"Balances", b"TotalIssuance", &mut para_ring_total_issuance) + .take_map(b"Balances", b"Locks", &mut para_ring_locks, get_hashed_key); + + log::info!("check solo ring locks, there should not be any `solo_ring_locks`"); + check_locks(solo_ring_locks); + log::info!("check solo kton locks, there should not be any `solo_kton_locks`"); + check_locks(solo_kton_locks); + log::info!("check para locks, there should not be any `para_ring_locks`"); + check_locks(para_ring_locks); + + (solo_ring_total_issuance + para_ring_total_issuance, kton_total_issuance) } } + +fn prune(locks: &mut Map) { + // https://github.dev/darwinia-network/darwinia-common/blob/6a9392cfb9fe2c99b1c2b47d0c36125d61991bb7/frame/staking/src/primitives.rs#L39 + const STAKING: [u8; 8] = *b"da/staki"; + // https://github.dev/darwinia-network/darwinia/blob/2d1c1436594b2c397d450e317c35eb16c71105d6/runtime/crab/src/pallets/elections_phragmen.rs#L8 + const PHRAGMEN_ELECTION: [u8; 8] = *b"phrelect"; + // https://github.dev/paritytech/substrate/blob/19162e43be45817b44c7d48e50d03f074f60fbf4/frame/democracy/src/lib.rs#L190 + const DEMOCRACY: [u8; 8] = *b"democrac"; + // https://github.dev/paritytech/substrate/blob/19162e43be45817b44c7d48e50d03f074f60fbf4/frame/vesting/src/lib.rs#L86 + const VESTING: [u8; 8] = *b"vesting "; + const RELAY_AUTHORITY: [u8; 8] = *b"ethrauth"; + // https://github.dev/darwinia-network/darwinia/blob/2d1c1436594b2c397d450e317c35eb16c71105d6/runtime/crab/src/pallets/fee_market.rs#L35 + const FEE_MARKET_0: [u8; 8] = *b"da/feelf"; + // https://github.dev/darwinia-network/darwinia/blob/2d1c1436594b2c397d450e317c35eb16c71105d6/runtime/crab/src/pallets/fee_market.rs#L36 + const FEE_MARKET_1: [u8; 8] = *b"da/feecp"; + // https://github.dev/darwinia-network/darwinia/blob/2d1c1436594b2c397d450e317c35eb16c71105d6/runtime/darwinia/src/pallets/fee_market.rs#L37 + const FEE_MARKET_2: [u8; 8] = *b"da/feedp"; + + locks.retain(|k, v| { + v.retain(|l| match l.id { + STAKING | PHRAGMEN_ELECTION | DEMOCRACY | VESTING | RELAY_AUTHORITY | FEE_MARKET_0 + | FEE_MARKET_1 | FEE_MARKET_2 => false, + id => { + log::error!( + "pruned unknown lock id({}) of account({})", + String::from_utf8_lossy(&id), + get_last_64(k) + ); + + false + }, + }); + + !v.is_empty() + }); +} + +fn check_locks(locks: Map) { + locks + .into_iter() + .for_each(|(k, _)| log::error!("found unexpected locks of account({})", get_last_64(&k))); +} diff --git a/tool/state-processor/src/main.rs b/tool/state-processor/src/main.rs index 79d76bf67..bb0bbbb6f 100644 --- a/tool/state-processor/src/main.rs +++ b/tool/state-processor/src/main.rs @@ -1,5 +1,6 @@ mod balances; mod system; +mod vesting; mod type_registry; use type_registry::*; @@ -45,6 +46,7 @@ impl Processor { fn process(mut self) -> Result<()> { self.process_system(); + self.process_vesting(); self.save() } @@ -108,7 +110,23 @@ impl State { self } - fn take( + fn take_value(&mut self, pallet: &[u8], item: &[u8], value: &mut D) -> &mut Self + where + D: Decode, + { + let key = item_key(pallet, item); + + if let Some(v) = self.0.remove(&key) { + match decode(&v) { + Ok(v) => *value = v, + Err(e) => log::warn!("failed to decode `{key}:{v}`, due to `{e}`"), + } + } + + self + } + + fn take_map( &mut self, pallet: &[u8], item: &[u8], @@ -119,6 +137,7 @@ impl State { D: Decode, F: Fn(&str, &str) -> String, { + let len = buffer.len(); let item_key = item_key(pallet, item); self.0.retain(|full_key, v| { @@ -136,6 +155,14 @@ impl State { } }); + if buffer.len() == len { + log::info!( + "no new item inserted for {}::{}", + String::from_utf8_lossy(pallet), + String::from_utf8_lossy(item) + ); + } + self } } @@ -157,7 +184,7 @@ where fn pallet_key(pallet: &[u8]) -> String { let prefix = subhasher::twox128(pallet); - array_bytes::bytes2hex("0x", &prefix) + array_bytes::bytes2hex("0x", prefix) } fn item_key(pallet: &[u8], item: &[u8]) -> String { @@ -166,6 +193,10 @@ fn item_key(pallet: &[u8], item: &[u8]) -> String { array_bytes::bytes2hex("0x", &k.0) } +fn full_key(pallet: &[u8], item: &[u8], hash: &str) -> String { + format!("{}{hash}", item_key(pallet, item)) +} + fn encode_value(v: V) -> String where V: Encode, @@ -182,13 +213,17 @@ where Ok(D::decode(&mut &*v)?) } -// twox128(pallet) + twox128(item) + blake2_256_concat(item_key) -> blake2_256_concat(item_key) -fn get_blake2_128_concat_suffix(full_key: &str, item_key: &str) -> String { +// twox128(pallet) + twox128(item) -> twox128(pallet) + twox128(item) +fn get_identity_key(key: &str, _: &str) -> String { + key.into() +} + +// twox128(pallet) + twox128(item) + *(item_key) -> *(item_key) +fn get_hashed_key(full_key: &str, item_key: &str) -> String { full_key.trim_start_matches(item_key).into() } -// twox128(pallet) + twox128(item) + blake2_256_concat(account_id_32) -> account_id_32 -#[allow(unused)] -fn get_concat_suffix(full_key: &str, _: &str) -> String { - format!("0x{}", &full_key[full_key.len() - 64..]) +// twox128(pallet) + twox128(item) + *_concat(account_id_32) -> account_id_32 +fn get_last_64(key: &str) -> String { + format!("0x{}", &key[key.len() - 64..]) } diff --git a/tool/state-processor/src/system/README.md b/tool/state-processor/src/system/README.md new file mode 100644 index 000000000..3ced4bf12 --- /dev/null +++ b/tool/state-processor/src/system/README.md @@ -0,0 +1,36 @@ +### Process steps +- take solo account infos and remaining balances + - merge solo remaining balances + - adjust solo balance decimals +- take para account infos +- process balances + - take solo balances total issuances and locks + - prune solo balance locks + - prune staking, phragmen election, democracy, vesting, relay authority, fee market locks + - check if there are any other locks + - adjust solo balances items' decimals + - take para balances total issuances and locks + - there should not be any locks on parachain +- use all previous data to build the new accounts and calculate total issuances + - merge solo and para account infos + - how to deal with the account references? - TODO +- set `Balances::TotalIssuance` + - compare the calculated one with the storage one + - remove para backing account - TODO + - check remaining sum - TODO + - XCM relate - TODO +- set KTON total issuances - TODO + - compare the calculated one with the storage one + - check remaining sum - TODO +- set accounts + - if is EVM address + - insert to `System::Account` + - if is Substrate address + - reset the nonce + - insert to `AccountMigration::Accounts` + - calculate misc frozen and fee frozen + +- some remaining accounts, bridge endpoint accounts - TODO +- special accounts - TODO + - parachain backing account 0x8c585F9791EE5b4B23fe82888cE576DBB69607eB + - bridge root account 0x726f6f7400000000000000000000000000000000 diff --git a/tool/state-processor/src/system/mod.rs b/tool/state-processor/src/system/mod.rs index 66977d038..ba0214515 100644 --- a/tool/state-processor/src/system/mod.rs +++ b/tool/state-processor/src/system/mod.rs @@ -22,80 +22,16 @@ impl Processor { // Balances storage items. // https://github.com/paritytech/substrate/blob/polkadot-v0.9.16/frame/balances/src/lib.rs#L486-L535 pub fn process_system(&mut self) -> &mut Self { + let solo_account_infos = self.process_solo_account_infos(); + let para_account_infos = self.process_para_account_infos(); + let (ring_total_issuance_storage, kton_total_issuance_storage) = self.process_balances(); let mut accounts = Map::default(); - let mut solo_account_infos = Map::default(); - let mut solo_ring_locks = Map::default(); - let mut solo_kton_locks = Map::default(); - let mut para_account_infos = Map::default(); - let mut remaining_ring = Map::default(); - let mut remaining_kton = Map::default(); - let mut ring_total_issuance = 0; - - log::info!("take solo and remaining balances"); - self.solo_state - .take::( - b"System", - b"Account", - &mut solo_account_infos, - get_blake2_128_concat_suffix, - ) - .take::( - b"Ethereum", - b"RemainingRingBalance", - &mut remaining_ring, - get_blake2_128_concat_suffix, - ) - .take::( - b"Ethereum", - b"RemainingKtonBalance", - &mut remaining_kton, - get_blake2_128_concat_suffix, - ); - - log::info!("take solo and para balance locks"); - self.process_balances(&mut solo_ring_locks, &mut solo_kton_locks); - - log::info!("take para balances"); - self.para_state.take::( - b"System", - b"Account", - &mut para_account_infos, - get_blake2_128_concat_suffix, - ); - - log::info!("adjust solo balance decimals"); - solo_account_infos.iter_mut().for_each(|(_, v)| { - v.data.free *= GWEI; - v.data.reserved *= GWEI; - v.data.free_kton_or_misc_frozen *= GWEI; - v.data.reserved_kton_or_fee_frozen *= GWEI; - }); - - log::info!("merge solo and remaining balances"); - remaining_ring.into_iter().for_each(|(k, v)| { - if let Some(a) = solo_account_infos.get_mut(&k) { - a.data.free += v; - } else { - log::warn!("`RemainingRingBalance({k})` not found"); - } - }); - remaining_kton.into_iter().for_each(|(k, v)| { - if let Some(a) = solo_account_infos.get_mut(&k) { - a.data.free_kton_or_misc_frozen += v; - } else { - log::warn!("`RemainingKtonBalance({k})` not found"); - } - }); + let mut ring_total_issuance = u128::default(); + let mut kton_total_issuance = u128::default(); log::info!("build accounts"); - log::info!("calculate ring total issuance"); + log::info!("calculate total issuance"); solo_account_infos.into_iter().for_each(|(k, v)| { - let ring_locks = solo_ring_locks.remove(&k).unwrap_or_default(); - let kton_locks = solo_kton_locks.remove(&k).unwrap_or_default(); - - ring_total_issuance += v.data.free; - ring_total_issuance += v.data.reserved; - accounts.insert( k.clone(), AccountAll { @@ -109,17 +45,19 @@ impl Processor { // --- ring: v.data.free, ring_reserved: v.data.reserved, - ring_locks, + ring_locks: Default::default(), kton: v.data.free_kton_or_misc_frozen, kton_reserved: v.data.reserved_kton_or_fee_frozen, - kton_locks, + kton_locks: Default::default(), }, ); - }); - para_account_infos.into_iter().for_each(|(k, v)| { + ring_total_issuance += v.data.free; ring_total_issuance += v.data.reserved; - + kton_total_issuance += v.data.free_kton_or_misc_frozen; + kton_total_issuance += v.data.reserved_kton_or_fee_frozen; + }); + para_account_infos.into_iter().for_each(|(k, v)| { accounts .entry(k.clone()) .and_modify(|a| { @@ -135,22 +73,27 @@ impl Processor { sufficients: v.sufficients, ring: v.data.free, ring_reserved: v.data.reserved, - ring_locks: Vec::new(), + ring_locks: Default::default(), kton: 0, kton_reserved: 0, - kton_locks: Vec::new(), + kton_locks: Default::default(), }); - }); - log::info!("check solo remaining locks"); - solo_ring_locks.into_iter().for_each(|(k, _)| log::warn!("ring_locks' owner({k}) dropped")); - solo_kton_locks.into_iter().for_each(|(k, _)| log::warn!("kton_locks' owner({k}) dropped")); + ring_total_issuance += v.data.free; + ring_total_issuance += v.data.reserved; + }); let state = &mut self.shell_chain_spec.genesis.raw.top; log::info!("set `Balances::TotalIssuance`"); + log::info!("ring_total_issuance({ring_total_issuance})"); + log::info!("ring_total_issuance_storage({ring_total_issuance_storage})"); state.insert(item_key(b"Balances", b"TotalIssuance"), encode_value(ring_total_issuance)); + log::info!("kton_total_issuance({kton_total_issuance})"); + log::info!("kton_total_issuance_storage({kton_total_issuance_storage})"); + // TODO: set KTON total issuance + log::info!("update ring misc frozen and fee frozen"); log::info!("set `System::Account`"); log::info!("set `Balances::Locks`"); @@ -163,38 +106,91 @@ impl Processor { data: AccountData { free: v.ring, reserved: v.ring_reserved, - free_kton_or_misc_frozen: 0, - reserved_kton_or_fee_frozen: 0, + free_kton_or_misc_frozen: Default::default(), + reserved_kton_or_fee_frozen: Default::default(), }, }; - // https://github.com/paritytech/substrate/blob/polkadot-v0.9.16/frame/balances/src/lib.rs#L945-L952 - // Update ring misc frozen and fee frozen. - for l in v.ring_locks.iter() { - if l.reasons == Reasons::All || l.reasons == Reasons::Misc { - a.data.free_kton_or_misc_frozen = a.data.free_kton_or_misc_frozen.max(l.amount); - } - if l.reasons == Reasons::All || l.reasons == Reasons::Fee { - a.data.reserved_kton_or_fee_frozen = - a.data.reserved_kton_or_fee_frozen.max(l.amount); - } + if is_evm_address(&k) { + state.insert(full_key(b"System", b"Account", &k), encode_value(a)); + + // TODO: migrate kton balances. + } else { + a.nonce = 0; + + state.insert(full_key(b"AccountMigration", b"Accounts", &k), encode_value(a)); } - // --- - // TODO: migrate kton locks. - // --- - - // Set `System::Account`. - state.insert(format!("{}{k}", item_key(b"System", b"Account")), encode_value(a)); - // Set `Balances::Locks`. - // Skip empty locks. - if !v.ring_locks.is_empty() { - state.insert( - format!("{}{k}", item_key(b"Balances", b"Locks")), - encode_value(v.ring_locks), + }); + + self + } + + fn process_solo_account_infos(&mut self) -> Map { + let mut account_infos = >::default(); + let mut remaining_ring = >::default(); + let mut remaining_kton = >::default(); + + log::info!("take solo account infos and remaining balances"); + self.solo_state + .take_map(b"System", b"Account", &mut account_infos, get_hashed_key) + .take_map(b"Ethereum", b"RemainingRingBalance", &mut remaining_ring, get_hashed_key) + .take_map(b"Ethereum", b"RemainingKtonBalance", &mut remaining_kton, get_hashed_key); + + log::info!("merge solo remaining balances"); + remaining_ring.into_iter().for_each(|(k, v)| { + if let Some(a) = account_infos.get_mut(&k) { + a.data.free += v; + } else { + log::error!( + "`Account({})` not found while merging `RemainingRingBalance`", + get_last_64(&k) + ); + } + }); + remaining_kton.into_iter().for_each(|(k, v)| { + if let Some(a) = account_infos.get_mut(&k) { + a.data.free_kton_or_misc_frozen += v; + } else { + log::error!( + "`Account({})` not found while merging `RemainingKtonBalance`", + get_last_64(&k) ); } }); - self + log::info!("adjust solo balance decimals"); + account_infos.iter_mut().for_each(|(_, v)| { + v.data.free *= GWEI; + v.data.reserved *= GWEI; + v.data.free_kton_or_misc_frozen *= GWEI; + v.data.reserved_kton_or_fee_frozen *= GWEI; + }); + + account_infos } + + fn process_para_account_infos(&mut self) -> Map { + let mut account_infos = >::default(); + + log::info!("take para account infos"); + self.para_state.take_map(b"System", b"Account", &mut account_infos, get_hashed_key); + + account_infos + } +} + +fn is_evm_address(address: &str) -> bool { + let address = array_bytes::hex2bytes_unchecked(address); + + address.starts_with(b"dvm:") + && address[1..31].iter().fold(address[0], |checksum, &byte| checksum ^ byte) == address[31] +} + +#[test] +fn verify_evm_address_checksum_should_work() { + // subalfred key 5ELRpquT7C3mWtjerpPfdmaGoSh12BL2gFCv2WczEcv6E1zL + // sub-seed + // public-key 0x64766d3a00000000000000b7de7f8c52ac75e036d05fda53a75cf12714a76973 + // Substrate 5ELRpquT7C3mWtjerpPfdmaGoSh12BL2gFCv2WczEcv6E1zL + assert!(is_evm_address("0x64766d3a00000000000000b7de7f8c52ac75e036d05fda53a75cf12714a76973")); } diff --git a/tool/state-processor/src/vesting/mod.rs b/tool/state-processor/src/vesting/mod.rs new file mode 100644 index 000000000..adcefb429 --- /dev/null +++ b/tool/state-processor/src/vesting/mod.rs @@ -0,0 +1,8 @@ +// darwinia +use crate::*; + +impl Processor { + pub fn process_vesting(&mut self) -> &mut Self { + self + } +}