Skip to content

Commit

Permalink
Rpc: filters performance improvement (solana-labs#20185)
Browse files Browse the repository at this point in the history 
* Add Base58,Base64,Bytes to MemcmpEncodedBytes

* Rpc: decode memcmp before filtering accounts

* Add deprecated attribute

* Add Memcmp::bytes

* Fix clippy for deprecated

* Another clippy fix

* merge RpcFilterError::DataTooLarge

* add deprecation for Base58DataTooLarge

* change filter data size limit

* strict data size len for base58

* add magic numbers

* fix tests
  • Loading branch information
@fanatid @jeffwashington
fanatid authored and jeffwashington committed Oct 15, 2021
1 parent 6eb3a9a commit e101ecc
Show file tree
Hide file tree
Showing 4 changed files with 147 additions and 53 deletions.
4 changes: 2 additions & 2 deletions cli/src/cluster_query.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1720,15 +1720,15 @@ pub fn process_show_stakes(
// Filter by `StakeState::Stake(_, _)`
rpc_filter::RpcFilterType::Memcmp(rpc_filter::Memcmp {
offset: 0,
bytes: rpc_filter::MemcmpEncodedBytes::Binary(
bytes: rpc_filter::MemcmpEncodedBytes::Base58(
bs58::encode([2, 0, 0, 0]).into_string(),
),
encoding: Some(rpc_filter::MemcmpEncoding::Binary),
}),
// Filter by `Delegation::voter_pubkey`, which begins at byte offset 124
rpc_filter::RpcFilterType::Memcmp(rpc_filter::Memcmp {
offset: 124,
bytes: rpc_filter::MemcmpEncodedBytes::Binary(
bytes: rpc_filter::MemcmpEncodedBytes::Base58(
vote_account_pubkeys[0].to_string(),
),
encoding: Some(rpc_filter::MemcmpEncoding::Binary),
Expand Down
14 changes: 7 additions & 7 deletions cli/src/program.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1148,18 +1148,18 @@ fn get_buffers(
) -> Result<CliUpgradeableBuffers, Box<dyn std::error::Error>> {
let mut filters = vec![RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 0, 0, 0]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 0, 0, 0]).into_string()),
encoding: None,
})];
if let Some(authority_pubkey) = authority_pubkey {
filters.push(RpcFilterType::Memcmp(Memcmp {
offset: ACCOUNT_TYPE_SIZE,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1]).into_string()),
encoding: None,
}));
filters.push(RpcFilterType::Memcmp(Memcmp {
offset: ACCOUNT_TYPE_SIZE + OPTION_SIZE,
bytes: MemcmpEncodedBytes::Binary(
bytes: MemcmpEncodedBytes::Base58(
bs58::encode(authority_pubkey.as_ref()).into_string(),
),
encoding: None,
Expand Down Expand Up @@ -1201,18 +1201,18 @@ fn get_programs(
) -> Result<CliUpgradeablePrograms, Box<dyn std::error::Error>> {
let mut filters = vec![RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 0, 0, 0]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 0, 0, 0]).into_string()),
encoding: None,
})];
if let Some(authority_pubkey) = authority_pubkey {
filters.push(RpcFilterType::Memcmp(Memcmp {
offset: ACCOUNT_TYPE_SIZE + SLOT_SIZE,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1]).into_string()),
encoding: None,
}));
filters.push(RpcFilterType::Memcmp(Memcmp {
offset: ACCOUNT_TYPE_SIZE + SLOT_SIZE + OPTION_SIZE,
bytes: MemcmpEncodedBytes::Binary(
bytes: MemcmpEncodedBytes::Base58(
bs58::encode(authority_pubkey.as_ref()).into_string(),
),
encoding: None,
Expand All @@ -1236,7 +1236,7 @@ fn get_programs(
bytes.extend_from_slice(programdata_address.as_ref());
let filters = vec![RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(bytes).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(bytes).into_string()),
encoding: None,
})];

Expand Down
133 changes: 104 additions & 29 deletions client/src/rpc_filter.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
use thiserror::Error;
#![allow(deprecated)]
use {std::borrow::Cow, thiserror::Error};

const MAX_DATA_SIZE: usize = 128;
const MAX_DATA_BASE58_SIZE: usize = 175;
const MAX_DATA_BASE64_SIZE: usize = 172;

#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
Expand All @@ -15,15 +20,50 @@ impl RpcFilterType {
let encoding = compare.encoding.as_ref().unwrap_or(&MemcmpEncoding::Binary);
match encoding {
MemcmpEncoding::Binary => {
let MemcmpEncodedBytes::Binary(bytes) = &compare.bytes;

if bytes.len() > 128 {
Err(RpcFilterError::Base58DataTooLarge)
} else {
bs58::decode(&bytes)
.into_vec()
.map(|_| ())
.map_err(|e| e.into())
use MemcmpEncodedBytes::*;
match &compare.bytes {
Binary(bytes) if bytes.len() > MAX_DATA_BASE58_SIZE => {
Err(RpcFilterError::Base58DataTooLarge)
}
Base58(bytes) if bytes.len() > MAX_DATA_BASE58_SIZE => {
Err(RpcFilterError::DataTooLarge)
}
Base64(bytes) if bytes.len() > MAX_DATA_BASE64_SIZE => {
Err(RpcFilterError::DataTooLarge)
}
Bytes(bytes) if bytes.len() > MAX_DATA_SIZE => {
Err(RpcFilterError::DataTooLarge)
}
_ => Ok(()),
}?;
match &compare.bytes {
Binary(bytes) => {
let bytes = bs58::decode(&bytes)
.into_vec()
.map_err(RpcFilterError::DecodeError)?;
if bytes.len() > MAX_DATA_SIZE {
Err(RpcFilterError::Base58DataTooLarge)
} else {
Ok(())
}
}
Base58(bytes) => {
let bytes = bs58::decode(&bytes).into_vec()?;
if bytes.len() > MAX_DATA_SIZE {
Err(RpcFilterError::DataTooLarge)
} else {
Ok(())
}
}
Base64(bytes) => {
let bytes = base64::decode(&bytes)?;
if bytes.len() > MAX_DATA_SIZE {
Err(RpcFilterError::DataTooLarge)
} else {
Ok(())
}
}
Bytes(_) => Ok(()),
}
}
}
Expand All @@ -34,10 +74,24 @@ impl RpcFilterType {

#[derive(Error, PartialEq, Debug)]
pub enum RpcFilterError {
#[error("bs58 decode error")]
DecodeError(#[from] bs58::decode::Error),
#[error("encoded binary data should be less than 129 bytes")]
DataTooLarge,
#[deprecated(
since = "1.9.0",
note = "Error for MemcmpEncodedBytes::Binary which is deprecated"
)]
#[error("encoded binary (base 58) data should be less than 129 bytes")]
Base58DataTooLarge,
#[deprecated(
since = "1.9.0",
note = "Error for MemcmpEncodedBytes::Binary which is deprecated"
)]
#[error("bs58 decode error")]
DecodeError(bs58::decode::Error),
#[error("base58 decode error")]
Base58DecodeError(#[from] bs58::decode::Error),
#[error("base64 decode error")]
Base64DecodeError(#[from] base64::DecodeError),
}

#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
Expand All @@ -49,7 +103,14 @@ pub enum MemcmpEncoding {
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "camelCase", untagged)]
pub enum MemcmpEncodedBytes {
#[deprecated(
since = "1.9.0",
note = "Please use MemcmpEncodedBytes::Base58 instead"
)]
Binary(String),
Base58(String),
Base64(String),
Bytes(Vec<u8>),
}

#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
Expand All @@ -63,14 +124,18 @@ pub struct Memcmp {
}

impl Memcmp {
pub fn bytes_match(&self, data: &[u8]) -> bool {
pub fn bytes(&self) -> Option<Cow<Vec<u8>>> {
use MemcmpEncodedBytes::*;
match &self.bytes {
MemcmpEncodedBytes::Binary(bytes) => {
let bytes = bs58::decode(bytes).into_vec();
if bytes.is_err() {
return false;
}
let bytes = bytes.unwrap();
Binary(bytes) | Base58(bytes) => bs58::decode(bytes).into_vec().ok().map(Cow::Owned),
Base64(bytes) => base64::decode(bytes).ok().map(Cow::Owned),
Bytes(bytes) => Some(Cow::Borrowed(bytes)),
}
}

pub fn bytes_match(&self, data: &[u8]) -> bool {
match self.bytes() {
Some(bytes) => {
if self.offset > data.len() {
return false;
}
Expand All @@ -79,6 +144,7 @@ impl Memcmp {
}
data[self.offset..self.offset + bytes.len()] == bytes[..]
}
None => false,
}
}
}
Expand All @@ -87,62 +153,71 @@ impl Memcmp {
mod tests {
use super::*;

#[test]
fn test_worst_case_encoded_tx_goldens() {
let ff_data = vec![0xffu8; MAX_DATA_SIZE];
let data58 = bs58::encode(&ff_data).into_string();
assert_eq!(data58.len(), MAX_DATA_BASE58_SIZE);
let data64 = base64::encode(&ff_data);
assert_eq!(data64.len(), MAX_DATA_BASE64_SIZE);
}

#[test]
fn test_bytes_match() {
let data = vec![1, 2, 3, 4, 5];

// Exact match of data succeeds
assert!(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 2, 3, 4, 5]).into_string()),
encoding: None,
}
.bytes_match(&data));

// Partial match of data succeeds
assert!(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![1, 2]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![1, 2]).into_string()),
encoding: None,
}
.bytes_match(&data));

// Offset partial match of data succeeds
assert!(Memcmp {
offset: 2,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 4]).into_string()),
encoding: None,
}
.bytes_match(&data));

// Incorrect partial match of data fails
assert!(!Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![2]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![2]).into_string()),
encoding: None,
}
.bytes_match(&data));

// Bytes overrun data fails
assert!(!Memcmp {
offset: 2,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![3, 4, 5, 6]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![3, 4, 5, 6]).into_string()),
encoding: None,
}
.bytes_match(&data));

// Offset outside data fails
assert!(!Memcmp {
offset: 6,
bytes: MemcmpEncodedBytes::Binary(bs58::encode(vec![5]).into_string()),
bytes: MemcmpEncodedBytes::Base58(bs58::encode(vec![5]).into_string()),
encoding: None,
}
.bytes_match(&data));

// Invalid base-58 fails
assert!(!Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary("III".to_string()),
bytes: MemcmpEncodedBytes::Base58("III".to_string()),
encoding: None,
}
.bytes_match(&data));
Expand All @@ -157,7 +232,7 @@ mod tests {
assert_eq!(
RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()),
bytes: MemcmpEncodedBytes::Base58(base58_bytes.to_string()),
encoding: None,
})
.verify(),
Expand All @@ -172,11 +247,11 @@ mod tests {
assert_eq!(
RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Binary(base58_bytes.to_string()),
bytes: MemcmpEncodedBytes::Base58(base58_bytes.to_string()),
encoding: None,
})
.verify(),
Err(RpcFilterError::Base58DataTooLarge)
Err(RpcFilterError::DataTooLarge)
);
}
}
Loading

0 comments on commit e101ecc

Please sign in to comment.