Skip to content

Commit

Permalink
Adding support for address lookup tables (#319)
Browse files Browse the repository at this point in the history
* Initial implementation of address lookup tables

* Implementing fetching of address lookup tables

* Use address lookup tables to calculate account prio fees

* Adding some logs and renaming classes

* preloading all the alts correctly

* Changes after groovies review

* Fixing issues after rebase
  • Loading branch information
godmodegalactus authored Feb 6, 2024
1 parent 7e7992b commit 196b400
Show file tree
Hide file tree
Showing 27 changed files with 544 additions and 82 deletions.
177 changes: 122 additions & 55 deletions Cargo.lock

Large diffs are not rendered by default.

6 changes: 5 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ members = [
"cluster-endpoints",
"blockstore",
"prioritization_fees",
"bench"
"bench",
"address_lookup_tables"
]

[workspace.package]
Expand All @@ -33,6 +34,8 @@ solana-streamer = "~1.17.15"
solana-account-decoder = "~1.17.15"
solana-ledger = "~1.17.15"
solana-program = "~1.17.15"
solana-address-lookup-table-program = "~1.17.15"

itertools = "0.10.5"
rangetools = "0.1.4"
serde = { version = "1.0.160", features = ["derive"] }
Expand Down Expand Up @@ -71,6 +74,7 @@ solana-lite-rpc-cluster-endpoints = {path = "cluster-endpoints", version="0.2.4"
solana-lite-rpc-blockstore = {path = "blockstore", version="0.2.4"}
solana-lite-rpc-stakevote = {path = "stake_vote", version="0.2.4"}
solana-lite-rpc-prioritization-fees = {path = "prioritization_fees", version="0.2.4"}
solana-lite-rpc-address-lookup-tables = {path = "address_lookup_tables", version="0.2.4"}

async-trait = "0.1.68"
yellowstone-grpc-client = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.12.0+solana.1.17.15" }
Expand Down
43 changes: 43 additions & 0 deletions address_lookup_tables/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
[package]
name = "solana-lite-rpc-address-lookup-tables"
version = "0.2.4"
edition = "2021"
description = "Library to save and update address lookup tables in lite-rpc"
rust-version = "1.73.0"
repository = "https://github.com/blockworks-foundation/lite-rpc"
license = "AGPL"

[dependencies]
solana-sdk = { workspace = true }
solana-rpc-client-api = { workspace = true }
solana-transaction-status = { workspace = true }
solana-version = { workspace = true }
solana-client = { workspace = true }
solana-net-utils = { workspace = true }
solana-pubsub-client = { workspace = true }
solana-rpc-client = { workspace = true }
solana-streamer = { workspace = true }
solana-account-decoder = { workspace = true }
solana-address-lookup-table-program = { workspace = true }

serde = { workspace = true }
serde_json = { workspace = true }
tokio = "1.*"
bincode = { workspace = true }
bs58 = { workspace = true }
base64 = { workspace = true }
thiserror = { workspace = true }
futures = { workspace = true }
bytes = { workspace = true }
anyhow = { workspace = true }
log = { workspace = true }
dashmap = { workspace = true }
quinn = { workspace = true }
chrono = { workspace = true }
rustls = { workspace = true }
async-trait = { workspace = true }
itertools = { workspace = true }
prometheus = { workspace = true }
lazy_static = { workspace = true }

solana-lite-rpc-core = { workspace = true }
229 changes: 229 additions & 0 deletions address_lookup_tables/src/address_lookup_table_store.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,229 @@
use async_trait::async_trait;
use dashmap::DashMap;
use itertools::Itertools;
use prometheus::{opts, register_int_gauge, IntGauge};
use serde::{Deserialize, Serialize};
use solana_address_lookup_table_program::state::AddressLookupTable;
use solana_lite_rpc_core::traits::address_lookup_table_interface::AddressLookupTableInterface;
use solana_rpc_client::nonblocking::rpc_client::RpcClient;
use solana_sdk::{commitment_config::CommitmentConfig, pubkey::Pubkey};
use std::{sync::Arc, time::Duration};

lazy_static::lazy_static! {
static ref LRPC_ALTS_IN_STORE: IntGauge =
register_int_gauge!(opts!("literpc_alts_stored", "Alts stored in literpc")).unwrap();
}

#[derive(Clone)]
pub struct AddressLookupTableStore {
rpc_client: Arc<RpcClient>,
pub map: Arc<DashMap<Pubkey, Vec<Pubkey>>>,
}

impl AddressLookupTableStore {
pub fn new(rpc_client: Arc<RpcClient>) -> Self {
Self {
rpc_client,
map: Arc::new(DashMap::new()),
}
}

async fn load_alts_list(&self, alts_list: &[Pubkey]) {
log::trace!("Preloading {} ALTs", alts_list.len());
for batches in alts_list.chunks(1000).map(|x| x.to_vec()) {
let tasks = batches.chunks(100).map(|batch| {
let batch = batch.to_vec();
let rpc_client = self.rpc_client.clone();
let this = self.clone();
tokio::spawn(async move {
let data = rpc_client
.get_multiple_accounts_with_commitment(
&batch,
CommitmentConfig::processed(),
)
.await;

match data {
Ok(multiple_accounts) => {
for (index, acc) in multiple_accounts.value.iter().enumerate() {
if let Some(acc) = acc {
this.save_account(&batch[index], &acc.data);
}
}
}
Err(e) => {
log::error!(
"error loading {} alts with error {}, skipping lookup for this batch and continue",
batch.len(),
e.to_string()
);
}
};
})
});
if tokio::time::timeout(Duration::from_secs(60), futures::future::join_all(tasks))
.await
.is_err()
{
log::error!(
"timeout loading {} alts, skipping lookup for this batch and continue",
alts_list.len()
);
}
}
LRPC_ALTS_IN_STORE.set(self.map.len() as i64);
}

pub fn save_account(&self, address: &Pubkey, data: &[u8]) {
let lookup_table = AddressLookupTable::deserialize(data).unwrap();
if self
.map
.insert(*address, lookup_table.addresses.to_vec())
.is_none()
{
LRPC_ALTS_IN_STORE.inc();
}
drop(lookup_table);
}

pub async fn reload_if_necessary(
&self,
alt_messages: &[&solana_sdk::message::v0::MessageAddressTableLookup],
) {
let accounts_to_load = alt_messages
.iter()
.filter_map(|alt| match self.map.get(&alt.account_key) {
Some(alt_data) => {
let size = alt_data.len() as u8;
if alt.readonly_indexes.iter().any(|x| *x >= size)
|| alt.writable_indexes.iter().any(|x| *x >= size)
{
Some(alt.account_key)
} else {
None
}
}
None => Some(alt.account_key),
})
.collect_vec();
self.load_alts_list(&accounts_to_load).await;
}

pub async fn reload_alt_account(&self, address: &Pubkey) {
log::info!("Reloading {address:?}");

let account = match self
.rpc_client
.get_account_with_commitment(address, CommitmentConfig::processed())
.await
{
Ok(acc) => acc.value,
Err(e) => {
log::error!(
"Error for fetching address lookup table {} error :{}",
address.to_string(),
e.to_string()
);
None
}
};
match account {
Some(account) => {
self.save_account(address, &account.data);
}
None => {
log::error!("Cannot find address lookup table {}", address.to_string());
}
}
}

async fn get_accounts_in_address_lookup_table(
&self,
alt: &Pubkey,
accounts: &[u8],
) -> Option<Vec<Pubkey>> {
let alt_account = self.map.get(alt);
match alt_account {
Some(alt_account) => Some(
accounts
.iter()
.map(|i| alt_account[*i as usize])
.collect_vec(),
),
None => {
log::error!("address lookup table {} was not found", alt);
None
}
}
}

pub async fn get_accounts(&self, alt: &Pubkey, accounts: &[u8]) -> Vec<Pubkey> {
match self
.get_accounts_in_address_lookup_table(alt, accounts)
.await
{
Some(x) => x,
None => {
// forget alt for now, start loading it for next blocks
// loading should be on its way
vec![]
}
}
}

pub fn serialize_binary(&self) -> Vec<u8> {
bincode::serialize::<BinaryALTData>(&BinaryALTData::new(&self.map)).unwrap()
}

/// To load binary ALT file at the startup
pub fn load_binary(&self, binary_data: Vec<u8>) {
let binary_alt_data = bincode::deserialize::<BinaryALTData>(&binary_data).unwrap();
for (alt, accounts) in binary_alt_data.data.iter() {
self.map.insert(*alt, accounts.clone());
}
}
}

#[derive(Serialize, Deserialize)]
pub struct BinaryALTData {
data: Vec<(Pubkey, Vec<Pubkey>)>,
}

impl BinaryALTData {
pub fn new(map: &Arc<DashMap<Pubkey, Vec<Pubkey>>>) -> Self {
let data = map
.iter()
.map(|x| (*x.key(), x.value().clone()))
.collect_vec();
Self { data }
}
}

#[async_trait]
impl AddressLookupTableInterface for AddressLookupTableStore {
async fn resolve_addresses_from_lookup_table(
&self,
message_address_table_lookup: &solana_sdk::message::v0::MessageAddressTableLookup,
) -> (Vec<Pubkey>, Vec<Pubkey>) {
(
self.get_accounts(
&message_address_table_lookup.account_key,
&message_address_table_lookup.writable_indexes,
)
.await,
self.get_accounts(
&message_address_table_lookup.account_key,
&message_address_table_lookup.readonly_indexes,
)
.await,
)
}

async fn reload_if_necessary(
&self,
message_address_table_lookups: &[&solana_sdk::message::v0::MessageAddressTableLookup],
) {
self.reload_if_necessary(message_address_table_lookups)
.await;
}
}
1 change: 1 addition & 0 deletions address_lookup_tables/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pub mod address_lookup_table_store;
2 changes: 1 addition & 1 deletion blockstore/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name = "solana-lite-rpc-blockstore"
version = "0.2.4"
edition = "2021"
description = "Store and proved blocks in PostgreSQL DB and via Yellowstone Faithful"
rust-version = "1.70.0"
rust-version = "1.73.0"
repository = "https://github.com/blockworks-foundation/lite-rpc"
license = "AGPL"

Expand Down
1 change: 1 addition & 0 deletions blockstore/src/block_stores/postgres/postgres_block.rs
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,7 @@ mod tests {
message: "message".to_string(),
writable_accounts: vec![],
readable_accounts: vec![],
address_lookup_tables: vec![],
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,7 @@ mod tests {
message: "some message".to_string(),
writable_accounts: vec![],
readable_accounts: vec![],
address_lookup_tables: vec![],
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ impl PostgresTransaction {
readable_accounts: vec![],
writable_accounts: vec![],
is_vote: false,
address_lookup_tables: vec![],
}
}

Expand Down
2 changes: 1 addition & 1 deletion cluster-endpoints/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name = "solana-lite-rpc-cluster-endpoints"
version = "0.2.4"
edition = "2021"
description = "Core classes and methods used by solana lite rpc"
rust-version = "1.70.0"
rust-version = "1.73.0"
repository = "https://github.com/blockworks-foundation/lite-rpc"
license = "AGPL"

Expand Down
6 changes: 6 additions & 0 deletions cluster-endpoints/src/grpc_subscription.rs
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,11 @@ pub fn from_grpc_block_update(
.map(|(_, pk)| *pk)
.collect();

let address_lookup_tables = message
.address_table_lookups()
.map(|x| x.to_vec())
.unwrap_or_default();

Some(TransactionInfo {
signature: signature.to_string(),
is_vote: is_vote_transaction,
Expand All @@ -216,6 +221,7 @@ pub fn from_grpc_block_update(
message: BASE64.encode(message.serialize()),
readable_accounts,
writable_accounts,
address_lookup_tables,
})
})
.collect();
Expand Down
7 changes: 7 additions & 0 deletions cluster-endpoints/src/rpc_polling/poll_blocks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,12 @@ pub fn from_ui_block(
}
}

let address_lookup_tables = tx
.message
.address_table_lookups()
.map(|x| x.to_vec())
.unwrap_or_default();

Some(TransactionInfo {
signature,
is_vote: is_vote_transaction,
Expand All @@ -297,6 +303,7 @@ pub fn from_ui_block(
message,
readable_accounts,
writable_accounts,
address_lookup_tables,
})
})
.collect();
Expand Down
2 changes: 1 addition & 1 deletion core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name = "solana-lite-rpc-core"
version = "0.2.4"
edition = "2021"
description = "Core classes and methods used by solana lite rpc"
rust-version = "1.70.0"
rust-version = "1.73.0"
repository = "https://github.com/blockworks-foundation/lite-rpc"
license = "AGPL"

Expand Down
Loading

0 comments on commit 196b400

Please sign in to comment.