Skip to content

Commit

Permalink
Restore getProgramAccounts spl-token secondary-index functionality (s…
Browse files Browse the repository at this point in the history 
…olana-labs#20993)

* Allow get_spl_token_X_filters to match on any encoding, and optimize earlier

* Remove redundant optimize calls

* Compress match statements

* Add method docs, including note to use optimize_filters before spl-token checks

* Add logs
  • Loading branch information
@CriesofCarrots @dankelleher
CriesofCarrots authored and dankelleher committed Nov 24, 2021
1 parent 5d52153 commit 67a89b4
Show file tree
Hide file tree
Showing 2 changed files with 56 additions and 29 deletions.
32 changes: 16 additions & 16 deletions client/src/rpc_filter.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,22 +22,11 @@ impl RpcFilterType {
MemcmpEncoding::Binary => {
use MemcmpEncodedBytes::*;
match &compare.bytes {
Binary(bytes) if bytes.len() > MAX_DATA_BASE58_SIZE => {
Err(RpcFilterError::Base58DataTooLarge)
}
Base58(bytes) if bytes.len() > MAX_DATA_BASE58_SIZE => {
Err(RpcFilterError::DataTooLarge)
}
Base64(bytes) if bytes.len() > MAX_DATA_BASE64_SIZE => {
Err(RpcFilterError::DataTooLarge)
}
Bytes(bytes) if bytes.len() > MAX_DATA_SIZE => {
Err(RpcFilterError::DataTooLarge)
}
_ => Ok(()),
}?;
match &compare.bytes {
// DEPRECATED
Binary(bytes) => {
if bytes.len() > MAX_DATA_BASE58_SIZE {
return Err(RpcFilterError::Base58DataTooLarge);
}
let bytes = bs58::decode(&bytes)
.into_vec()
.map_err(RpcFilterError::DecodeError)?;
Expand All @@ -48,6 +37,9 @@ impl RpcFilterType {
}
}
Base58(bytes) => {
if bytes.len() > MAX_DATA_BASE58_SIZE {
return Err(RpcFilterError::DataTooLarge);
}
let bytes = bs58::decode(&bytes).into_vec()?;
if bytes.len() > MAX_DATA_SIZE {
Err(RpcFilterError::DataTooLarge)
Expand All @@ -56,14 +48,22 @@ impl RpcFilterType {
}
}
Base64(bytes) => {
if bytes.len() > MAX_DATA_BASE64_SIZE {
return Err(RpcFilterError::DataTooLarge);
}
let bytes = base64::decode(&bytes)?;
if bytes.len() > MAX_DATA_SIZE {
Err(RpcFilterError::DataTooLarge)
} else {
Ok(())
}
}
Bytes(_) => Ok(()),
Bytes(bytes) => {
if bytes.len() > MAX_DATA_SIZE {
return Err(RpcFilterError::DataTooLarge);
}
Ok(())
}
}
}
}
Expand Down
53 changes: 40 additions & 13 deletions rpc/src/rpc.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ use {
fee_calculator::FeeCalculator,
hash::Hash,
message::{Message, SanitizedMessage},
pubkey::Pubkey,
pubkey::{Pubkey, PUBKEY_BYTES},
signature::{Keypair, Signature, Signer},
stake::state::{StakeActivationStatus, StakeState},
stake_history::StakeHistory,
Expand Down Expand Up @@ -377,14 +377,15 @@ impl JsonRpcRequestProcessor {
&self,
program_id: &Pubkey,
config: Option<RpcAccountInfoConfig>,
filters: Vec<RpcFilterType>,
mut filters: Vec<RpcFilterType>,
with_context: bool,
) -> Result<OptionalContext<Vec<RpcKeyedAccount>>> {
let config = config.unwrap_or_default();
let bank = self.bank(config.commitment);
let encoding = config.encoding.unwrap_or(UiAccountEncoding::Binary);
let data_slice_config = config.data_slice;
check_slice_and_encoding(&encoding, data_slice_config.is_some())?;
optimize_filters(&mut filters);
let keyed_accounts = {
if let Some(owner) = get_spl_token_owner_filter(program_id, &filters) {
self.get_filtered_spl_token_accounts_by_owner(&bank, &owner, filters)?
Expand Down Expand Up @@ -1872,7 +1873,6 @@ impl JsonRpcRequestProcessor {
index_key: owner_key.to_string(),
});
}
optimize_filters(&mut filters);
Ok(bank
.get_filtered_indexed_accounts(&IndexKey::SplTokenOwner(*owner_key), |account| {
account.owner() == &spl_token_id_v2_0()
Expand Down Expand Up @@ -1921,7 +1921,6 @@ impl JsonRpcRequestProcessor {
index_key: mint_key.to_string(),
});
}
optimize_filters(&mut filters);
Ok(bank
.get_filtered_indexed_accounts(&IndexKey::SplTokenMint(*mint_key), |account| {
account.owner() == &spl_token_id_v2_0()
Expand Down Expand Up @@ -2151,58 +2150,86 @@ fn encode_account<T: ReadableAccount>(
}
}

/// Analyze custom filters to determine if the result will be a subset of spl-token accounts by
/// owner.
/// NOTE: `optimize_filters()` should almost always be called before using this method because of
/// the strict match on `MemcmpEncodedBytes::Bytes`.
fn get_spl_token_owner_filter(program_id: &Pubkey, filters: &[RpcFilterType]) -> Option<Pubkey> {
if program_id != &spl_token_id_v2_0() {
return None;
}
let mut data_size_filter: Option<u64> = None;
let mut owner_key: Option<Pubkey> = None;
let mut incorrect_owner_len: Option<usize> = None;
for filter in filters {
match filter {
RpcFilterType::DataSize(size) => data_size_filter = Some(*size),
RpcFilterType::Memcmp(Memcmp {
offset: SPL_TOKEN_ACCOUNT_OWNER_OFFSET,
bytes: MemcmpEncodedBytes::Base58(bytes),
bytes: MemcmpEncodedBytes::Bytes(bytes),
..
}) => {
if let Ok(key) = Pubkey::from_str(bytes) {
owner_key = Some(key)
if bytes.len() == PUBKEY_BYTES {
owner_key = Some(Pubkey::new(bytes));
} else {
incorrect_owner_len = Some(bytes.len());
}
}
_ => {}
}
}
if data_size_filter == Some(TokenAccount::get_packed_len() as u64) {
if let Some(incorrect_owner_len) = incorrect_owner_len {
info!(
"Incorrect num bytes ({:?}) provided for spl_token_owner_filter",
incorrect_owner_len
);
}
owner_key
} else {
debug!("spl_token program filters do not match by-owner index requisites");
None
}
}

/// Analyze custom filters to determine if the result will be a subset of spl-token accounts by
/// mint.
/// NOTE: `optimize_filters()` should almost always be called before using this method because of
/// the strict match on `MemcmpEncodedBytes::Bytes`.
fn get_spl_token_mint_filter(program_id: &Pubkey, filters: &[RpcFilterType]) -> Option<Pubkey> {
if program_id != &spl_token_id_v2_0() {
return None;
}
let mut data_size_filter: Option<u64> = None;
let mut mint: Option<Pubkey> = None;
let mut incorrect_mint_len: Option<usize> = None;
for filter in filters {
match filter {
RpcFilterType::DataSize(size) => data_size_filter = Some(*size),
RpcFilterType::Memcmp(Memcmp {
offset: SPL_TOKEN_ACCOUNT_MINT_OFFSET,
bytes: MemcmpEncodedBytes::Base58(bytes),
bytes: MemcmpEncodedBytes::Bytes(bytes),
..
}) => {
if let Ok(key) = Pubkey::from_str(bytes) {
mint = Some(key)
if bytes.len() == PUBKEY_BYTES {
mint = Some(Pubkey::new(bytes));
} else {
incorrect_mint_len = Some(bytes.len());
}
}
_ => {}
}
}
if data_size_filter == Some(TokenAccount::get_packed_len() as u64) {
if let Some(incorrect_mint_len) = incorrect_mint_len {
info!(
"Incorrect num bytes ({:?}) provided for spl_token_mint_filter",
incorrect_mint_len
);
}
mint
} else {
debug!("spl_token program filters do not match by-mint index requisites");
None
}
}
Expand Down Expand Up @@ -7673,7 +7700,7 @@ pub mod tests {
&[
RpcFilterType::Memcmp(Memcmp {
offset: 32,
bytes: MemcmpEncodedBytes::Base58(owner.to_string()),
bytes: MemcmpEncodedBytes::Bytes(owner.to_bytes().to_vec()),
encoding: None
}),
RpcFilterType::DataSize(165)
Expand All @@ -7689,7 +7716,7 @@ pub mod tests {
&[
RpcFilterType::Memcmp(Memcmp {
offset: 0,
bytes: MemcmpEncodedBytes::Base58(owner.to_string()),
bytes: MemcmpEncodedBytes::Bytes(owner.to_bytes().to_vec()),
encoding: None
}),
RpcFilterType::DataSize(165)
Expand All @@ -7703,7 +7730,7 @@ pub mod tests {
&[
RpcFilterType::Memcmp(Memcmp {
offset: 32,
bytes: MemcmpEncodedBytes::Base58(owner.to_string()),
bytes: MemcmpEncodedBytes::Bytes(owner.to_bytes().to_vec()),
encoding: None
}),
RpcFilterType::DataSize(165)
Expand Down

0 comments on commit 67a89b4

Please sign in to comment.