merge queue: embarking main (77bd1e3) and #4776 together

.changelog/unreleased/improvements/4776-query-conversions-requires-masp-epoch.md

@@ -0,0 +1,12 @@
+- Querying the conversion state becomes more difficult as nodes progress
+  to higher and higher epochs. For instance, on a mainnet clone running on
+  accelerated epochs, client queries for conversions always timeout (even after
+  modifying timeout_broadcast_tx_commit) when the node goes beyond a certain
+  MASP epoch. This happens because the conversion state grows linearly with
+  the MASP epoch counter. This PR attempts to address this problem by making
+  the conversions RPC endpoint require that clients specify the MASP epoch
+  they want conversions from. This implies two things: first the size of the
+  response from the conversions RPC endpoint should now be constant (equal to
+  the number of tokens in the shielded rewards program), and second a client
+  requiring all conversions now has to do a separate query for each MASP epoch.
+  ([\#4776](https://github.com/namada-net/namada/pull/4776))

crates/apps_lib/src/client/masp.rs

@@ -117,13 +117,37 @@ pub async fn syncing<
 
     #[cfg(feature = "testing")]
     {
+        use std::collections::BTreeMap;
+
+        use futures::StreamExt;
+        use namada_core::masp::MaspEpoch;
+
         // Load the confirmed context (if present) and update the conversions
         // for the shielded history. This is needed for integration
         // tests only as the cli wallet is not supposed to compile the
         // history of shielded transactions
         shielded.load_confirmed().await;
-        for (asset_type, (token, denom, position, epoch, _conv)) in
-            namada_sdk::rpc::query_conversions(&client).await?
+        let current_masp_epoch =
+            namada_sdk::rpc::query_masp_epoch(&client).await?;
+        let epochs: Vec<_> = MaspEpoch::iter_bounds_inclusive(
+            MaspEpoch::zero(),
+            current_masp_epoch,
+        )
+        .collect();
+        let conversion_tasks = epochs
+            .iter()
+            .map(|epoch| namada_sdk::rpc::query_conversions(&client, epoch));
+        let conversions = futures::stream::iter(conversion_tasks)
+            .buffer_unordered(100)
+            .fold(BTreeMap::default(), async |mut acc, conversion| {
+                acc.append(
+                    &mut conversion.expect("Conversion should be defined"),
+                );
+                acc
+            })
+            .await;
+
+        for (asset_type, (token, denom, position, epoch, _conv)) in conversions
         {
             let pre_asset_type = AssetData {
                 token,

crates/apps_lib/src/client/rpc.rs

@@ -6,6 +6,7 @@ use std::io;
 use color_eyre::owo_colors::OwoColorize;
 use data_encoding::HEXLOWER;
 use either::Either;
+use futures::StreamExt;
 use masp_primitives::asset_type::AssetType;
 use masp_primitives::merkle_tree::MerklePath;
 use masp_primitives::sapling::Node;
@@ -2143,16 +2144,31 @@ pub async fn query_conversions(
         .wallet()
         .await
         .get_addresses_with_vp_type(AddressVpType::Token);
-    let conversions = rpc::query_conversions(context.client())
+
+    // Download conversions from all epochs to facilitate decoding asset types
+    let from = MaspEpoch::zero();
+    let to = rpc::query_masp_epoch(context.client())
         .await
-        .expect("Conversions should be defined");
+        .expect("Unable to query current MASP epoch");
+    let epochs: Vec<_> = MaspEpoch::iter_bounds_inclusive(from, to).collect();
+    let conversion_tasks = epochs
+        .iter()
+        .map(|epoch| rpc::query_conversions(context.client(), epoch));
+    let conversions = futures::stream::iter(conversion_tasks)
+        .buffer_unordered(100)
+        .fold(BTreeMap::default(), async |mut acc, conversion| {
+            acc.append(&mut conversion.expect("Conversion should be defined"));
+            acc
+        })
+        .await;
 
     if args.dump_tree {
         display_line!(context.io(), "Conversions: {conversions:?}");
     }
 
     // Track whether any non-sentinel conversions are found
     let mut conversions_found = false;
+
     for (addr, _denom, digit, epoch, amt) in conversions.values() {
         // If the user has specified any targets, then meet them
         // If we have a sentinel conversion, then skip printing

crates/sdk/src/queries/shell.rs

@@ -98,7 +98,7 @@ router! {SHELL,
     ( "conv" / [asset_type: AssetType] ) -> Option<Conversion> = read_conversion,
 
     // Conversion state access - read conversion
-    ( "conversions" ) -> BTreeMap<AssetType, ConversionWithoutPath> = read_conversions,
+    ( "conversions" / [masp_epoch: MaspEpoch] ) -> BTreeMap<AssetType, ConversionWithoutPath> = read_conversions,
 
     // Conversion state access - read conversion
     ( "masp_reward_tokens" ) -> Vec<MaspTokenRewardData> = masp_reward_tokens,
@@ -211,6 +211,7 @@ where
 /// Query to read the conversion state
 fn read_conversions<D, H, V, T>(
     ctx: RequestCtx<'_, D, H, V, T>,
+    masp_epoch: MaspEpoch,
 ) -> namada_storage::Result<BTreeMap<AssetType, ConversionWithoutPath>>
 where
     D: 'static + DB + for<'iter> DBIter<'iter> + Sync,
@@ -222,6 +223,7 @@ where
         .conversion_state
         .assets
         .iter()
+        .filter(|&(_, asset)| (asset.epoch == masp_epoch))
         .map(|(&asset_type, asset)| {
             (
                 asset_type,

crates/sdk/src/rpc.rs

@@ -390,6 +390,7 @@ pub async fn query_conversion<C: namada_io::Client + Sync>(
 /// Query conversions
 pub async fn query_conversions<C: namada_io::Client + Sync>(
     client: &C,
+    masp_epoch: &MaspEpoch,
 ) -> Result<
     BTreeMap<
         AssetType,
@@ -403,7 +404,9 @@ pub async fn query_conversions<C: namada_io::Client + Sync>(
     >,
     error::Error,
 > {
-    convert_response::<C, _>(RPC.shell().read_conversions(client).await)
+    convert_response::<C, _>(
+        RPC.shell().read_conversions(client, masp_epoch).await,
+    )
 }
 
 /// Query the total rewards minted by MASP