diff --git a/cli/src/cluster_query.rs b/cli/src/cluster_query.rs index c2d5e9a221fa7b..4918cb5bf881d1 100644 --- a/cli/src/cluster_query.rs +++ b/cli/src/cluster_query.rs @@ -1721,7 +1721,7 @@ pub fn process_show_stakes( // Filter by `StakeState::Stake(_, _)` rpc_filter::RpcFilterType::Memcmp(rpc_filter::Memcmp { offset: 0, - bytes: rpc_filter::MemcmpEncodedBytes::Binary( + bytes: rpc_filter::MemcmpEncodedBytes::Base58( bs58::encode([2, 0, 0, 0]).into_string(), ), encoding: Some(rpc_filter::MemcmpEncoding::Binary), @@ -1729,7 +1729,7 @@ pub fn process_show_stakes( // Filter by `Delegation::voter_pubkey`, which begins at byte offset 124 rpc_filter::RpcFilterType::Memcmp(rpc_filter::Memcmp { offset: 124, - bytes: rpc_filter::MemcmpEncodedBytes::Binary( + bytes: rpc_filter::MemcmpEncodedBytes::Base58( vote_account_pubkeys[0].to_string(), ), encoding: Some(rpc_filter::MemcmpEncoding::Binary), diff --git a/cli/src/program.rs b/cli/src/program.rs index 67e770460fe969..6fd310eef70d92 100644 --- a/cli/src/program.rs +++ b/cli/src/program.rs @@ -1146,18 +1146,18 @@ fn get_buffers( ) -> Result> { let mut filters = vec![RpcFilterType::Memcmp(Memcmp { offset: 0, - bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 0, 0, 0]).into_string()), + bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 0, 0, 0]).into_string()), encoding: None, })]; if let Some(authority_pubkey) = authority_pubkey { filters.push(RpcFilterType::Memcmp(Memcmp { offset: ACCOUNT_TYPE_SIZE, - bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1]).into_string()), + bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1]).into_string()), encoding: None, })); filters.push(RpcFilterType::Memcmp(Memcmp { offset: ACCOUNT_TYPE_SIZE + OPTION_SIZE, - bytes: MemcmpEncodedBytes::Binary( + bytes: MemcmpEncodedBytes::Base58( bs58::encode(authority_pubkey.as_ref()).into_string(), ), encoding: None, @@ -1199,18 +1199,18 @@ fn get_programs( ) -> Result> { let mut filters = vec![RpcFilterType::Memcmp(Memcmp { offset: 0, - bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 0, 0, 0]).into_string()), + bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 0, 0, 0]).into_string()), encoding: None, })]; if let Some(authority_pubkey) = authority_pubkey { filters.push(RpcFilterType::Memcmp(Memcmp { offset: ACCOUNT_TYPE_SIZE + SLOT_SIZE, - bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1]).into_string()), + bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1]).into_string()), encoding: None, })); filters.push(RpcFilterType::Memcmp(Memcmp { offset: ACCOUNT_TYPE_SIZE + SLOT_SIZE + OPTION_SIZE, - bytes: MemcmpEncodedBytes::Binary( + bytes: MemcmpEncodedBytes::Base58( bs58::encode(authority_pubkey.as_ref()).into_string(), ), encoding: None, @@ -1234,7 +1234,7 @@ fn get_programs( bytes.extend_from_slice(programdata_address.as_ref()); let filters = vec![RpcFilterType::Memcmp(Memcmp { offset: 0, - bytes: MemcmpEncodedBytes::Binary(bs58::encode(bytes).into_string()), + bytes: MemcmpEncodedBytes::Base58(bs58::encode(bytes).into_string()), encoding: None, })]; diff --git a/client/src/rpc_filter.rs b/client/src/rpc_filter.rs index eeb54e5dcfd085..0944c24c56dc37 100644 --- a/client/src/rpc_filter.rs +++ b/client/src/rpc_filter.rs @@ -1,4 +1,9 @@ -use thiserror::Error; +#![allow(deprecated)] +use {std::borrow::Cow, thiserror::Error}; + +const MAX_DATA_SIZE: usize = 128; +const MAX_DATA_BASE58_SIZE: usize = 175; +const MAX_DATA_BASE64_SIZE: usize = 172; #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -15,15 +20,50 @@ impl RpcFilterType { let encoding = compare.encoding.as_ref().unwrap_or(&MemcmpEncoding::Binary); match encoding { MemcmpEncoding::Binary => { - let MemcmpEncodedBytes::Binary(bytes) = &compare.bytes; - - if bytes.len() > 128 { - Err(RpcFilterError::Base58DataTooLarge) - } else { - bs58::decode(&bytes) - .into_vec() - .map(|_| ()) - .map_err(|e| e.into()) + use MemcmpEncodedBytes::*; + match &compare.bytes { + Binary(bytes) if bytes.len() > MAX_DATA_BASE58_SIZE => { + Err(RpcFilterError::Base58DataTooLarge) + } + Base58(bytes) if bytes.len() > MAX_DATA_BASE58_SIZE => { + Err(RpcFilterError::DataTooLarge) + } + Base64(bytes) if bytes.len() > MAX_DATA_BASE64_SIZE => { + Err(RpcFilterError::DataTooLarge) + } + Bytes(bytes) if bytes.len() > MAX_DATA_SIZE => { + Err(RpcFilterError::DataTooLarge) + } + _ => Ok(()), + }?; + match &compare.bytes { + Binary(bytes) => { + let bytes = bs58::decode(&bytes) + .into_vec() + .map_err(RpcFilterError::DecodeError)?; + if bytes.len() > MAX_DATA_SIZE { + Err(RpcFilterError::Base58DataTooLarge) + } else { + Ok(()) + } + } + Base58(bytes) => { + let bytes = bs58::decode(&bytes).into_vec()?; + if bytes.len() > MAX_DATA_SIZE { + Err(RpcFilterError::DataTooLarge) + } else { + Ok(()) + } + } + Base64(bytes) => { + let bytes = base64::decode(&bytes)?; + if bytes.len() > MAX_DATA_SIZE { + Err(RpcFilterError::DataTooLarge) + } else { + Ok(()) + } + } + Bytes(_) => Ok(()), } } } @@ -34,10 +74,24 @@ impl RpcFilterType { #[derive(Error, PartialEq, Debug)] pub enum RpcFilterError { - #[error("bs58 decode error")] - DecodeError(#[from] bs58::decode::Error), + #[error("encoded binary data should be less than 129 bytes")] + DataTooLarge, + #[deprecated( + since = "1.9.0", + note = "Error for MemcmpEncodedBytes::Binary which is deprecated" + )] #[error("encoded binary (base 58) data should be less than 129 bytes")] Base58DataTooLarge, + #[deprecated( + since = "1.9.0", + note = "Error for MemcmpEncodedBytes::Binary which is deprecated" + )] + #[error("bs58 decode error")] + DecodeError(bs58::decode::Error), + #[error("base58 decode error")] + Base58DecodeError(#[from] bs58::decode::Error), + #[error("base64 decode error")] + Base64DecodeError(#[from] base64::DecodeError), } #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -49,7 +103,14 @@ pub enum MemcmpEncoding { #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase", untagged)] pub enum MemcmpEncodedBytes { + #[deprecated( + since = "1.9.0", + note = "Please use MemcmpEncodedBytes::Base58 instead" + )] Binary(String), + Base58(String), + Base64(String), + Bytes(Vec), } #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -63,14 +124,18 @@ pub struct Memcmp { } impl Memcmp { - pub fn bytes_match(&self, data: &[u8]) -> bool { + pub fn bytes(&self) -> Option>> { + use MemcmpEncodedBytes::*; match &self.bytes { - MemcmpEncodedBytes::Binary(bytes) => { - let bytes = bs58::decode(bytes).into_vec(); - if bytes.is_err() { - return false; - } - let bytes = bytes.unwrap(); + Binary(bytes) | Base58(bytes) => bs58::decode(bytes).into_vec().ok().map(Cow::Owned), + Base64(bytes) => base64::decode(bytes).ok().map(Cow::Owned), + Bytes(bytes) => Some(Cow::Borrowed(bytes)), + } + } + + pub fn bytes_match(&self, data: &[u8]) -> bool { + match self.bytes() { + Some(bytes) => { if self.offset > data.len() { return false; } @@ -79,6 +144,7 @@ impl Memcmp { } data[self.offset..self.offset + bytes.len()] == bytes[..] } + None => false, } } } @@ -87,6 +153,15 @@ impl Memcmp { mod tests { use super::*; + #[test] + fn test_worst_case_encoded_tx_goldens() { + let ff_data = vec![0xffu8; MAX_DATA_SIZE]; + let data58 = bs58::encode(&ff_data).into_string(); + assert_eq!(data58.len(), MAX_DATA_BASE58_SIZE); + let data64 = base64::encode(&ff_data); + assert_eq!(data64.len(), MAX_DATA_BASE64_SIZE); + } + #[test] fn test_bytes_match() { let data = vec![1, 2, 3, 4, 5]; @@ -94,7 +169,7 @@ mod tests { // Exact match of data succeeds assert!(Memcmp { offset: 0, - bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()), + bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()), encoding: None, } .bytes_match(&data)); @@ -102,7 +177,7 @@ mod tests { // Partial match of data succeeds assert!(Memcmp { offset: 0, - bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2]).into_string()), + bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 2]).into_string()), encoding: None, } .bytes_match(&data)); @@ -110,7 +185,7 @@ mod tests { // Offset partial match of data succeeds assert!(Memcmp { offset: 2, - bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4]).into_string()), + bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 4]).into_string()), encoding: None, } .bytes_match(&data)); @@ -118,7 +193,7 @@ mod tests { // Incorrect partial match of data fails assert!(!Memcmp { offset: 0, - bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![2]).into_string()), + bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![2]).into_string()), encoding: None, } .bytes_match(&data)); @@ -126,7 +201,7 @@ mod tests { // Bytes overrun data fails assert!(!Memcmp { offset: 2, - bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4, 5, 6]).into_string()), + bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 4, 5, 6]).into_string()), encoding: None, } .bytes_match(&data)); @@ -134,7 +209,7 @@ mod tests { // Offset outside data fails assert!(!Memcmp { offset: 6, - bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![5]).into_string()), + bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![5]).into_string()), encoding: None, } .bytes_match(&data)); @@ -142,7 +217,7 @@ mod tests { // Invalid base-58 fails assert!(!Memcmp { offset: 0, - bytes: MemcmpEncodedBytes::Binary("III".to_string()), + bytes: MemcmpEncodedBytes::Base58("III".to_string()), encoding: None, } .bytes_match(&data)); @@ -157,7 +232,7 @@ mod tests { assert_eq!( RpcFilterType::Memcmp(Memcmp { offset: 0, - bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()), + bytes: MemcmpEncodedBytes::Base58(base58_bytes.to_string()), encoding: None, }) .verify(), @@ -172,11 +247,11 @@ mod tests { assert_eq!( RpcFilterType::Memcmp(Memcmp { offset: 0, - bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()), + bytes: MemcmpEncodedBytes::Base58(base58_bytes.to_string()), encoding: None, }) .verify(), - Err(RpcFilterError::Base58DataTooLarge) + Err(RpcFilterError::DataTooLarge) ); } } diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index 9c0e20d3337d83..46b3da415c89a3 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -1688,7 +1688,7 @@ impl JsonRpcRequestProcessor { // Optional filter on Mint address filters.push(RpcFilterType::Memcmp(Memcmp { offset: 0, - bytes: MemcmpEncodedBytes::Binary(mint.to_string()), + bytes: MemcmpEncodedBytes::Bytes(mint.to_bytes().into()), encoding: None, })); } @@ -1732,15 +1732,13 @@ impl JsonRpcRequestProcessor { // Filter on Delegate is_some() RpcFilterType::Memcmp(Memcmp { offset: 72, - bytes: MemcmpEncodedBytes::Binary( - bs58::encode(bincode::serialize(&1u32).unwrap()).into_string(), - ), + bytes: MemcmpEncodedBytes::Bytes(bincode::serialize(&1u32).unwrap()), encoding: None, }), // Filter on Delegate address RpcFilterType::Memcmp(Memcmp { offset: 76, - bytes: MemcmpEncodedBytes::Binary(delegate.to_string()), + bytes: MemcmpEncodedBytes::Bytes(delegate.to_bytes().into()), encoding: None, }), ]; @@ -1779,8 +1777,9 @@ impl JsonRpcRequestProcessor { &self, bank: &Arc, program_id: &Pubkey, - filters: Vec, + mut filters: Vec, ) -> RpcCustomResult> { + optimize_filters(&mut filters); let filter_closure = |account: &AccountSharedData| { filters.iter().all(|filter_type| match filter_type { RpcFilterType::DataSize(size) => account.data().len() as u64 == *size, @@ -1837,7 +1836,7 @@ impl JsonRpcRequestProcessor { // Filter on Owner address filters.push(RpcFilterType::Memcmp(Memcmp { offset: SPL_TOKEN_ACCOUNT_OWNER_OFFSET, - bytes: MemcmpEncodedBytes::Binary(owner_key.to_string()), + bytes: MemcmpEncodedBytes::Bytes(owner_key.to_bytes().into()), encoding: None, })); @@ -1851,6 +1850,7 @@ impl JsonRpcRequestProcessor { index_key: owner_key.to_string(), }); } + optimize_filters(&mut filters); Ok(bank .get_filtered_indexed_accounts(&IndexKey::SplTokenOwner(*owner_key), |account| { account.owner() == &spl_token_id_v2_0() @@ -1886,7 +1886,7 @@ impl JsonRpcRequestProcessor { // Filter on Mint address filters.push(RpcFilterType::Memcmp(Memcmp { offset: SPL_TOKEN_ACCOUNT_MINT_OFFSET, - bytes: MemcmpEncodedBytes::Binary(mint_key.to_string()), + bytes: MemcmpEncodedBytes::Bytes(mint_key.to_bytes().into()), encoding: None, })); if self @@ -1899,6 +1899,7 @@ impl JsonRpcRequestProcessor { index_key: mint_key.to_string(), }); } + optimize_filters(&mut filters); Ok(bank .get_filtered_indexed_accounts(&IndexKey::SplTokenMint(*mint_key), |account| { account.owner() == &spl_token_id_v2_0() @@ -1916,6 +1917,24 @@ impl JsonRpcRequestProcessor { } } +fn optimize_filters(filters: &mut Vec) { + filters.iter_mut().for_each(|filter_type| { + if let RpcFilterType::Memcmp(compare) = filter_type { + use MemcmpEncodedBytes::*; + match &compare.bytes { + #[allow(deprecated)] + Binary(bytes) | Base58(bytes) => { + compare.bytes = Bytes(bs58::decode(bytes).into_vec().unwrap()); + } + Base64(bytes) => { + compare.bytes = Bytes(base64::decode(bytes).unwrap()); + } + _ => {} + } + } + }) +} + fn verify_transaction( transaction: &Transaction, libsecp256k1_0_5_upgrade_enabled: bool, @@ -2082,7 +2101,7 @@ fn get_spl_token_owner_filter(program_id: &Pubkey, filters: &[RpcFilterType]) -> RpcFilterType::DataSize(size) => data_size_filter = Some(*size), RpcFilterType::Memcmp(Memcmp { offset: SPL_TOKEN_ACCOUNT_OWNER_OFFSET, - bytes: MemcmpEncodedBytes::Binary(bytes), + bytes: MemcmpEncodedBytes::Base58(bytes), .. }) => { if let Ok(key) = Pubkey::from_str(bytes) { @@ -2110,7 +2129,7 @@ fn get_spl_token_mint_filter(program_id: &Pubkey, filters: &[RpcFilterType]) -> RpcFilterType::DataSize(size) => data_size_filter = Some(*size), RpcFilterType::Memcmp(Memcmp { offset: SPL_TOKEN_ACCOUNT_MINT_OFFSET, - bytes: MemcmpEncodedBytes::Binary(bytes), + bytes: MemcmpEncodedBytes::Base58(bytes), .. }) => { if let Ok(key) = Pubkey::from_str(bytes) { @@ -5895,7 +5914,7 @@ pub mod tests { fn test_rpc_verify_filter() { let filter = RpcFilterType::Memcmp(Memcmp { offset: 0, - bytes: MemcmpEncodedBytes::Binary( + bytes: MemcmpEncodedBytes::Base58( "13LeFbG6m2EP1fqCj9k66fcXsoTHMMtgr7c78AivUrYD".to_string(), ), encoding: None, @@ -5904,7 +5923,7 @@ pub mod tests { // Invalid base-58 let filter = RpcFilterType::Memcmp(Memcmp { offset: 0, - bytes: MemcmpEncodedBytes::Binary("III".to_string()), + bytes: MemcmpEncodedBytes::Base58("III".to_string()), encoding: None, }); assert!(verify_filter(&filter).is_err()); @@ -7400,7 +7419,7 @@ pub mod tests { &[ RpcFilterType::Memcmp(Memcmp { offset: 32, - bytes: MemcmpEncodedBytes::Binary(owner.to_string()), + bytes: MemcmpEncodedBytes::Base58(owner.to_string()), encoding: None }), RpcFilterType::DataSize(165) @@ -7416,7 +7435,7 @@ pub mod tests { &[ RpcFilterType::Memcmp(Memcmp { offset: 0, - bytes: MemcmpEncodedBytes::Binary(owner.to_string()), + bytes: MemcmpEncodedBytes::Base58(owner.to_string()), encoding: None }), RpcFilterType::DataSize(165) @@ -7430,7 +7449,7 @@ pub mod tests { &[ RpcFilterType::Memcmp(Memcmp { offset: 32, - bytes: MemcmpEncodedBytes::Binary(owner.to_string()), + bytes: MemcmpEncodedBytes::Base58(owner.to_string()), encoding: None }), RpcFilterType::DataSize(165)