Skip to content

Add tx order number and pagination #1930

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+483 −78
Open

Add tx order number and pagination #1930

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 45 additions & 7 deletions api-server/api-server-common/src/storage/impls/in_memory/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use crate::storage::storage_api::{
block_aux_data::{BlockAuxData, BlockWithExtraData},
AmountWithDecimals, ApiServerStorageError, BlockInfo, CoinOrTokenStatistic, Delegation,
FungibleTokenData, LockedUtxo, NftWithOwner, Order, PoolBlockStats, PoolDataWithExtraInfo,
TransactionInfo, Utxo, UtxoLock, UtxoWithExtraInfo,
TransactionInfo, TransactionWithBlockInfo, Utxo, UtxoLock, UtxoWithExtraInfo,
};
use common::{
address::Address,
Expand All @@ -29,7 +29,7 @@ use common::{
Block, ChainConfig, DelegationId, Destination, Genesis, OrderId, PoolId, Transaction,
UtxoOutPoint,
},
primitives::{id::WithId, Amount, BlockHeight, CoinOrTokenId, Id},
primitives::{id::WithId, Amount, BlockHeight, CoinOrTokenId, Id, Idable},
};
use std::{
cmp::Reverse,
Expand All @@ -51,6 +51,7 @@ struct ApiServerInMemoryStorage {
main_chain_blocks_table: BTreeMap<BlockHeight, Id<Block>>,
pool_data_table: BTreeMap<PoolId, BTreeMap<BlockHeight, PoolDataWithExtraInfo>>,
transaction_table: BTreeMap<Id<Transaction>, (Id<Block>, TransactionInfo)>,
order_transaction_table: BTreeMap<u64, Id<Transaction>>,
utxo_table: BTreeMap<UtxoOutPoint, BTreeMap<BlockHeight, Utxo>>,
address_utxos: BTreeMap<String, BTreeSet<UtxoOutPoint>>,
locked_utxo_table: BTreeMap<UtxoOutPoint, BTreeMap<BlockHeight, LockedUtxo>>,
Expand Down Expand Up @@ -78,6 +79,7 @@ impl ApiServerInMemoryStorage {
main_chain_blocks_table: BTreeMap::new(),
pool_data_table: BTreeMap::new(),
transaction_table: BTreeMap::new(),
order_transaction_table: BTreeMap::new(),
utxo_table: BTreeMap::new(),
address_utxos: BTreeMap::new(),
locked_utxo_table: BTreeMap::new(),
Expand Down Expand Up @@ -198,7 +200,7 @@ impl ApiServerInMemoryStorage {
&self,
len: u32,
offset: u32,
) -> Result<Vec<(BlockAuxData, TransactionInfo)>, ApiServerStorageError> {
) -> Result<Vec<TransactionWithBlockInfo>, ApiServerStorageError> {
Ok(self
.main_chain_blocks_table
.values()
Expand All @@ -208,13 +210,21 @@ impl ApiServerInMemoryStorage {
let block = self.block_table.get(block_id).expect("must exist");
block.block.transactions().iter().zip(block.tx_additional_infos.iter()).map(
|(tx, additinal_data)| {
(
*block_aux,
TransactionInfo {
let order_number = self
.order_transaction_table
.iter()
.find(|(_, tx_id)| **tx_id == tx.transaction().get_id())
.expect("must exist")
.0;

TransactionWithBlockInfo {
tx_info: TransactionInfo {
tx: tx.clone(),
additional_info: additinal_data.clone(),
},
)
block_aux: *block_aux,
order_number: *order_number,
}
},
)
})
Expand All @@ -223,6 +233,32 @@ impl ApiServerInMemoryStorage {
.collect())
}

fn get_transactions_with_block_by_order_number(
&self,
len: u32,
order_number: u64,
) -> Result<Vec<TransactionWithBlockInfo>, ApiServerStorageError> {
Ok(self
.order_transaction_table
.range(..order_number)
.rev()
.take(len as usize)
.map(|(order_number, tx_id)| {
let (block_id, tx_info) = self.transaction_table.get(tx_id).expect("must exist");
let block_aux = self.block_aux_data_table.get(block_id).expect("must exist");
TransactionWithBlockInfo {
tx_info: tx_info.clone(),
block_aux: *block_aux,
order_number: *order_number,
}
})
.collect())
}

fn get_last_transaction_order_number(&self) -> Result<u64, ApiServerStorageError> {
Ok(self.order_transaction_table.keys().last().copied().unwrap_or(0))
}

#[allow(clippy::type_complexity)]
fn get_transaction_with_block(
&self,
Expand Down Expand Up @@ -971,6 +1007,7 @@ impl ApiServerInMemoryStorage {
fn set_transaction(
&mut self,
transaction_id: Id<Transaction>,
order_number: u64,
owning_block: Id<Block>,
transaction: &TransactionInfo,
) -> Result<(), ApiServerStorageError> {
Expand All @@ -983,6 +1020,7 @@ impl ApiServerInMemoryStorage {

self.transaction_table
.insert(transaction_id, (owning_block, transaction.clone()));
self.order_transaction_table.insert(order_number, transaction_id);
Ok(())
}

Expand Down
17 changes: 15 additions & 2 deletions api-server/api-server-common/src/storage/impls/in_memory/transactional/read.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ use common::{
use crate::storage::storage_api::{
block_aux_data::BlockAuxData, AmountWithDecimals, ApiServerStorageError, ApiServerStorageRead,
BlockInfo, CoinOrTokenStatistic, Delegation, FungibleTokenData, NftWithOwner, Order,
PoolBlockStats, PoolDataWithExtraInfo, TransactionInfo, Utxo, UtxoWithExtraInfo,
PoolBlockStats, PoolDataWithExtraInfo, TransactionInfo, TransactionWithBlockInfo, Utxo,
UtxoWithExtraInfo,
};

use super::ApiServerInMemoryStorageTransactionalRo;
Expand Down Expand Up @@ -92,10 +93,22 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRo<'_> {
&self,
len: u32,
offset: u32,
) -> Result<Vec<(BlockAuxData, TransactionInfo)>, ApiServerStorageError> {
) -> Result<Vec<TransactionWithBlockInfo>, ApiServerStorageError> {
self.transaction.get_transactions_with_block(len, offset)
}

async fn get_transactions_with_block_by_order_number(
&self,
len: u32,
order_number: u64,
) -> Result<Vec<TransactionWithBlockInfo>, ApiServerStorageError> {
self.transaction.get_transactions_with_block_by_order_number(len, order_number)
}

async fn get_last_transaction_order_number(&self) -> Result<u64, ApiServerStorageError> {
self.transaction.get_last_transaction_order_number()
}

async fn get_delegation(
&self,
delegation_id: DelegationId,
Expand Down
21 changes: 18 additions & 3 deletions api-server/api-server-common/src/storage/impls/in_memory/transactional/write.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ use crate::storage::storage_api::{
block_aux_data::{BlockAuxData, BlockWithExtraData},
AmountWithDecimals, ApiServerStorageError, ApiServerStorageRead, ApiServerStorageWrite,
BlockInfo, CoinOrTokenStatistic, Delegation, FungibleTokenData, LockedUtxo, NftWithOwner,
Order, PoolBlockStats, PoolDataWithExtraInfo, TransactionInfo, Utxo, UtxoWithExtraInfo,
Order, PoolBlockStats, PoolDataWithExtraInfo, TransactionInfo, TransactionWithBlockInfo, Utxo,
UtxoWithExtraInfo,
};
use common::{
address::Address,
Expand Down Expand Up @@ -125,10 +126,12 @@ impl ApiServerStorageWrite for ApiServerInMemoryStorageTransactionalRw<'_> {
async fn set_transaction(
&mut self,
transaction_id: Id<Transaction>,
order_number: u64,
owning_block: Id<Block>,
transaction: &TransactionInfo,
) -> Result<(), ApiServerStorageError> {
self.transaction.set_transaction(transaction_id, owning_block, transaction)
self.transaction
.set_transaction(transaction_id, order_number, owning_block, transaction)
}

async fn set_block_aux_data(
Expand Down Expand Up @@ -399,10 +402,22 @@ impl ApiServerStorageRead for ApiServerInMemoryStorageTransactionalRw<'_> {
&self,
len: u32,
offset: u32,
) -> Result<Vec<(BlockAuxData, TransactionInfo)>, ApiServerStorageError> {
) -> Result<Vec<TransactionWithBlockInfo>, ApiServerStorageError> {
self.transaction.get_transactions_with_block(len, offset)
}

async fn get_transactions_with_block_by_order_number(
&self,
len: u32,
order_number: u64,
) -> Result<Vec<TransactionWithBlockInfo>, ApiServerStorageError> {
self.transaction.get_transactions_with_block_by_order_number(len, order_number)
}

async fn get_last_transaction_order_number(&self) -> Result<u64, ApiServerStorageError> {
self.transaction.get_last_transaction_order_number()
}

async fn get_pool_data(
&self,
pool_id: PoolId,
Expand Down
2 changes: 1 addition & 1 deletion api-server/api-server-common/src/storage/impls/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

pub const CURRENT_STORAGE_VERSION: u32 = 21;
pub const CURRENT_STORAGE_VERSION: u32 = 22;

pub mod in_memory;
pub mod postgres;
112 changes: 104 additions & 8 deletions api-server/api-server-common/src/storage/impls/postgres/queries.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ use crate::storage::{
block_aux_data::{BlockAuxData, BlockWithExtraData},
AmountWithDecimals, ApiServerStorageError, BlockInfo, CoinOrTokenStatistic, Delegation,
FungibleTokenData, LockedUtxo, NftWithOwner, Order, PoolBlockStats, PoolDataWithExtraInfo,
TransactionInfo, Utxo, UtxoWithExtraInfo,
TransactionInfo, TransactionWithBlockInfo, Utxo, UtxoWithExtraInfo,
},
};

Expand Down Expand Up @@ -83,6 +83,11 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> {
.map_err(|_| ApiServerStorageError::TimestampTooHigh(block_timestamp))
}

fn order_number_to_postgres_friendly(order_number: u64) -> i64 {
// Postgres doesn't like u64, so we have to convert it to i64
order_number.try_into().unwrap_or_else(|e| panic!("Invalid block height: {e}"))
}

pub async fn is_initialized(&mut self) -> Result<bool, ApiServerStorageError> {
let query_str = Self::get_table_exists_query("misc_data");
let row_count = self
Expand Down Expand Up @@ -670,6 +675,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> {
self.just_execute(
"CREATE TABLE ml.transactions (
transaction_id bytea PRIMARY KEY,
order_number bigint NOT NULL,
owning_block_id bytea NOT NULL REFERENCES ml.blocks(block_id),
transaction_data bytea NOT NULL
);", // block_id can be null if the transaction is not in the main chain
Expand Down Expand Up @@ -1777,7 +1783,7 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> {
&self,
len: u32,
offset: u32,
) -> Result<Vec<(BlockAuxData, TransactionInfo)>, ApiServerStorageError> {
) -> Result<Vec<TransactionWithBlockInfo>, ApiServerStorageError> {
let len = len as i64;
let offset = offset as i64;
let rows = self
Expand All @@ -1786,7 +1792,8 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> {
r#"
SELECT
t.transaction_data,
b.aux_data
b.aux_data,
t.order_number
FROM
ml.blocks mb
INNER JOIN
Expand All @@ -1807,30 +1814,117 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> {
.map(|data| {
let transaction_data: Vec<u8> = data.get(0);
let block_data: Vec<u8> = data.get(1);
let order_number: i64 = data.get(2);

let block_data =
let block_aux =
BlockAuxData::decode_all(&mut block_data.as_slice()).map_err(|e| {
ApiServerStorageError::DeserializationError(format!(
"Block deserialization failed: {}",
e
))
})?;

let transaction = TransactionInfo::decode_all(&mut transaction_data.as_slice())
let tx_info = TransactionInfo::decode_all(&mut transaction_data.as_slice())
.map_err(|e| {
ApiServerStorageError::DeserializationError(format!(
"Transaction deserialization failed: {e}"
))
})?;

Ok((block_data, transaction))
Ok(TransactionWithBlockInfo {
tx_info,
block_aux,
order_number: order_number as u64,
})
})
.collect()
}

pub async fn get_transactions_with_block_by_order_number(
&self,
len: u32,
order_number: u64,
) -> Result<Vec<TransactionWithBlockInfo>, ApiServerStorageError> {
let len = len as i64;
let order_number = Self::order_number_to_postgres_friendly(order_number);
let rows = self
.tx
.query(
r#"
SELECT
t.transaction_data,
b.aux_data,
t.order_number
FROM
ml.transactions t
INNER JOIN
ml.block_aux_data b ON t.owning_block_id = b.block_id
WHERE t.order_number < $1
ORDER BY t.order_number DESC
LIMIT $2;
"#,
&[&order_number, &len],
)
.await
.map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?;

rows.into_iter()
.map(|data| {
let transaction_data: Vec<u8> = data.get(0);
let block_data: Vec<u8> = data.get(1);
let order_number: i64 = data.get(2);

let block_aux =
BlockAuxData::decode_all(&mut block_data.as_slice()).map_err(|e| {
ApiServerStorageError::DeserializationError(format!(
"Block deserialization failed: {}",
e
))
})?;

let tx_info = TransactionInfo::decode_all(&mut transaction_data.as_slice())
.map_err(|e| {
ApiServerStorageError::DeserializationError(format!(
"Transaction deserialization failed: {e}"
))
})?;

Ok(TransactionWithBlockInfo {
tx_info,
block_aux,
order_number: order_number as u64,
})
})
.collect()
}

pub async fn get_last_transaction_order_number(&self) -> Result<u64, ApiServerStorageError> {
let row = self
.tx
.query_opt(
r#"
SELECT
MAX(t.order_number)
FROM
ml.transactions t;
"#,
&[],
)
.await
.map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?;

if let Some(row) = row {
let last_order_number: i64 = row.get(0);
Ok(last_order_number as u64)
} else {
Ok(0)
}
}

pub async fn set_transaction(
&mut self,
transaction_id: Id<Transaction>,
order_number: u64,
owning_block: Id<Block>,
transaction: &TransactionInfo,
) -> Result<(), ApiServerStorageError> {
Expand All @@ -1839,11 +1933,13 @@ impl<'a, 'b> QueryFromConnection<'a, 'b> {
transaction_id,
owning_block
);
let order_number = Self::order_number_to_postgres_friendly(order_number);

self.tx.execute(
"INSERT INTO ml.transactions (transaction_id, owning_block_id, transaction_data) VALUES ($1, $2, $3)
"INSERT INTO ml.transactions (transaction_id, owning_block_id, transaction_data, order_number) VALUES ($1, $2, $3, $4)
ON CONFLICT (transaction_id) DO UPDATE
SET owning_block_id = $2, transaction_data = $3;", &[&transaction_id.encode(), &owning_block.encode(), &transaction.encode()]
SET owning_block_id = $2, transaction_data = $3, order_number = $4;",
&[&transaction_id.encode(), &owning_block.encode(), &transaction.encode(), &order_number]
).await
.map_err(|e| ApiServerStorageError::LowLevelStorageError(e.to_string()))?;

Expand Down
Loading