Merge branch 'main' into gateway-rpc

README.md

@@ -15,11 +15,12 @@ The structure of this repository is the following:
 - `crates` contains Subspace-specific Rust crates used to build node and farmer, most are following Substrate naming conventions
   - `subspace-node` is an implementation of the node for Subspace protocol
   - `subspace-farmer` is a CLI farmer app
+  - `subspace-gateway` is a Subspace Distributed Storage Network gateway
 - `domains` contains client and runtime code for decoupled execution and domains
 - `shared` contains low-level primitives used by the node, farmer, and other applications
 
 ## How to run
 
-Please refer to [farming.md](/docs/farming.md) on how to run farmer. 
+Please refer to [farming.md](/docs/farming.md) for how to run the farmer.
 
 If you are looking to farm offline, or build from source for development purposes please refer to [development.md](/docs/development.md).

crates/pallet-subspace/Cargo.toml

@@ -31,7 +31,7 @@ subspace-verification = { version = "0.1.0", path = "../subspace-verification",
 
 [dev-dependencies]
 env_logger = "0.11.5"
-futures = "0.3.30"
+futures = "0.3.31"
 pallet-balances = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42" }
 rand = { version = "0.8.5", features = ["min_const_gen"] }
 sp-io = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42" }

crates/sc-consensus-subspace-rpc/Cargo.toml

@@ -14,7 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 
 [dependencies]
 async-oneshot = "0.5.9"
-futures = "0.3.30"
+futures = "0.3.31"
 futures-timer = "3.0.3"
 jsonrpsee = { version = "0.24.5", features = ["server", "macros"] }
 parking_lot = "0.12.2"

crates/sc-consensus-subspace/Cargo.toml

@@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 [dependencies]
 async-trait = "0.1.83"
 codec = { package = "parity-scale-codec", version = "3.6.12", features = ["derive"] }
-futures = "0.3.30"
+futures = "0.3.31"
 parking_lot = "0.12.2"
 rand = "0.8.5"
 rand_chacha = "0.3.1"

crates/sc-proof-of-time/Cargo.toml

@@ -13,7 +13,7 @@ include = [
 [dependencies]
 core_affinity = "0.8.1"
 derive_more = { version = "1.0.0", features = ["full"] }
-futures = "0.3.30"
+futures = "0.3.31"
 parity-scale-codec = { version = "3.6.12", features = ["derive"] }
 parking_lot = "0.12.2"
 rayon = "1.10.0"

crates/sc-subspace-block-relay/Cargo.toml

@@ -15,7 +15,7 @@ async-channel = "1.9.0"
 async-trait = "0.1.83"
 codec = { package = "parity-scale-codec", version = "3.6.12", default-features = false, features = ["derive"] }
 derive_more = { version = "1.0.0", features = ["full"] }
-futures = "0.3.30"
+futures = "0.3.31"
 parking_lot = "0.12.2"
 sc-client-api = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42" }
 sc-network = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42" }

crates/sp-domains-fraud-proof/Cargo.toml

@@ -49,7 +49,7 @@ fp-rpc = { version = "3.0.0-dev", git = "https://github.com/autonomys/frontier",
 fp-self-contained = { version = "1.0.0-dev", git = "https://github.com/autonomys/frontier", rev = "f80f9e2bad338f3bf3854b256b3c4edea23e5968", features = ['default'] }
 frame-support = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42" }
 frame-system = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42" }
-futures = "0.3.30"
+futures = "0.3.31"
 libsecp256k1 = { version = "0.7.1", features = ["static-context", "hmac"] }
 pallet-balances = { git = "https://github.com/subspace/polkadot-sdk", rev = "5871818e1d736f1843eb9078f886290695165c42" }
 pallet-ethereum = { git = "https://github.com/autonomys/frontier", rev = "f80f9e2bad338f3bf3854b256b3c4edea23e5968", features = ['default'] }

crates/subspace-farmer-components/Cargo.toml

@@ -23,7 +23,7 @@ backoff = { version = "0.4.0", features = ["futures", "tokio"] }
 bitvec = "1.0.1"
 # TODO: Switch to fs4 once https://github.com/al8n/fs4-rs/issues/15 is resolved
 fs2 = "0.4.3"
-futures = "0.3.30"
+futures = "0.3.31"
 hex = "0.4.3"
 libc = "0.2.159"
 parity-scale-codec = "3.6.12"
@@ -48,7 +48,7 @@ winapi = "0.3.9"
 
 [dev-dependencies]
 criterion = "0.5.1"
-futures = "0.3.30"
+futures = "0.3.31"
 subspace-archiving = { version = "0.1.0", path = "../subspace-archiving" }
 subspace-proof-of-space = { version = "0.1.0", path = "../subspace-proof-of-space" }
 

crates/subspace-farmer/Cargo.toml

@@ -33,7 +33,7 @@ event-listener = "5.3.1"
 event-listener-primitives = "2.0.1"
 fdlimit = { version = "0.3.0", optional = true }
 fs4 = "0.9.1"
-futures = "0.3.30"
+futures = "0.3.31"
 hex = { version = "0.4.3", features = ["serde"] }
 hwlocality = { version = "1.0.0-alpha.6", features = ["vendored"], optional = true }
 jsonrpsee = { version = "0.24.5", features = ["ws-client"] }

crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs

@@ -34,9 +34,9 @@ pub(crate) struct AuditOptions {
     #[arg(long)]
     with_single: bool,
     /// Size of PER FARM thread pool used for farming (mostly for blocking I/O, but also for some
-    /// compute-intensive operations during proving), defaults to number of logical CPUs
-    /// available on UMA system and number of logical CPUs in first NUMA node on NUMA system, but
-    /// not more than 32 threads
+    /// compute-intensive operations during proving). Defaults to the number of logical CPUs
+    /// on UMA systems, or the number of logical CPUs in the first NUMA node on NUMA systems, but
+    /// limited to 32 threads.
     #[arg(long)]
     farming_thread_pool_size: Option<NonZeroUsize>,
     /// Disk farm to audit
@@ -57,9 +57,9 @@ pub(crate) struct ProveOptions {
     #[arg(long)]
     with_single: bool,
     /// Size of PER FARM thread pool used for farming (mostly for blocking I/O, but also for some
-    /// compute-intensive operations during proving), defaults to number of logical CPUs
-    /// available on UMA system and number of logical CPUs in first NUMA node on NUMA system, but
-    /// not more than 32 threads
+    /// compute-intensive operations during proving). Defaults to the number of logical CPUs
+    /// on UMA systems, or the number of logical CPUs in the first NUMA node on NUMA systems, but
+    /// limited to 32 threads.
     #[arg(long)]
     farming_thread_pool_size: Option<NonZeroUsize>,
     /// Disk farm to prove

crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster.rs

@@ -52,7 +52,7 @@ struct SharedArgs {
     /// which can be done by starting NATS server with config file containing `max_payload = 2MB`.
     #[arg(long = "nats-server", required = true)]
     nats_servers: Vec<ServerAddr>,
-    /// Defines endpoints for the prometheus metrics server. It doesn't start without at least
+    /// Endpoints for the prometheus metrics server. It doesn't start without at least
     /// one specified endpoint. Format: 127.0.0.1:8080
     #[arg(long, aliases = ["metrics-endpoint", "metrics-endpoints"])]
     prometheus_listen_on: Vec<SocketAddr>,

crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller.rs

@@ -45,7 +45,7 @@ const GET_PIECE_MAX_INTERVAL: Duration = Duration::from_secs(40);
 pub(super) struct ControllerArgs {
     /// Piece getter concurrency.
     ///
-    /// Increase will result in higher memory usage.
+    /// Increasing this value will cause higher memory usage.
     #[arg(long, default_value = "128")]
     piece_getter_concurrency: NonZeroUsize,
     /// Base path where to store P2P network identity

crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller/farms.rs

@@ -14,7 +14,7 @@ use futures::{select, FutureExt, StreamExt};
 use parking_lot::Mutex;
 use std::collections::hash_map::Entry;
 use std::collections::{HashMap, VecDeque};
-use std::future::{pending, ready, Future};
+use std::future::{ready, Future};
 use std::mem;
 use std::pin::{pin, Pin};
 use std::sync::Arc;
@@ -139,9 +139,7 @@ pub(super) async fn maintain_farms(
     // Farm that is being added/removed right now (if any)
     let mut farm_add_remove_in_progress = (Box::pin(ready(None)) as AddRemoveFuture).fuse();
     // Initialize with pending future so it never ends
-    let mut farms = FuturesUnordered::from_iter([
-        Box::pin(pending()) as Pin<Box<dyn Future<Output = (FarmIndex, anyhow::Result<()>)>>>
-    ]);
+    let mut farms = FuturesUnordered::new();
 
     let farmer_identify_subscription = pin!(nats_client
         .subscribe_to_broadcasts::<ClusterFarmerIdentifyFarmBroadcast>(None, None)
@@ -253,7 +251,7 @@ pub(super) async fn maintain_farms(
             }
             result = farm_add_remove_in_progress => {
                 if let Some((farm_index, expired_receiver, farm)) = result {
-                    farms.push(Box::pin(async move {
+                    farms.push(async move {
                         select! {
                             result = farm.run().fuse() => {
                                 (farm_index, result)
@@ -263,7 +261,7 @@ pub(super) async fn maintain_farms(
                                 (farm_index, Ok(()))
                             }
                         }
-                    }));
+                    });
                 }
             }
         }

crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/farmer.rs

@@ -46,13 +46,13 @@ pub(super) const FARMER_IDENTIFICATION_BROADCAST_INTERVAL: Duration = Duration::
 /// Arguments for farmer
 #[derive(Debug, Parser)]
 pub(super) struct FarmerArgs {
-    /// One or more farm located at specified path, each with its own allocated space.
+    /// One or more farms located at specified paths, each with its own allocated space.
     ///
     /// In case of multiple disks, it is recommended to specify them individually rather than using
     /// RAID 0, that way farmer will be able to better take advantage of concurrency of individual
     /// drives.
     ///
-    /// Format for each farm is coma-separated list of strings like this:
+    /// The format for each farm is a coma-separated list of strings like this:
     ///
     ///   path=/path/to/directory,size=5T
     ///
@@ -65,33 +65,33 @@ pub(super) struct FarmerArgs {
     /// Address for farming rewards
     #[arg(long, value_parser = parse_ss58_reward_address)]
     reward_address: PublicKey,
-    /// Run temporary farmer with specified farm size in human-readable format (e.g. 10GB, 2TiB) or
+    /// Run a temporary farmer with a farm size in human-readable format (e.g. 10GB, 2TiB) or
     /// just bytes (e.g. 4096), this will create a temporary directory that will be deleted at the
     /// end of the process.
     #[arg(long, conflicts_with = "disk_farms")]
     tmp: Option<ByteSize>,
-    /// Maximum number of pieces in sector (can override protocol value to something lower).
+    /// Maximum number of pieces in a sector (can override protocol value to something lower).
     ///
     /// This will make plotting of individual sectors faster, decrease load on CPU proving, but also
     /// proportionally increase amount of disk reads during audits since every sector needs to be
     /// audited and there will be more of them.
     ///
-    /// This is primarily for development and not recommended to use by regular users.
+    /// This is primarily for development and not recommended for regular users.
     #[arg(long)]
     max_pieces_in_sector: Option<u16>,
     /// Do not print info about configured farms on startup
     #[arg(long)]
     no_info: bool,
-    /// Defines max number sectors farmer will encode concurrently, defaults to 50. Might be limited
+    /// The maximum number sectors a farmer will encode concurrently, defaults to 50. Might be limited
     /// by plotting capacity available in the cluster.
     ///
-    /// Increase will result in higher memory usage.
+    /// Increasing this value will cause higher memory usage.
     #[arg(long, default_value = "50")]
     sector_encoding_concurrency: NonZeroUsize,
     /// Size of PER FARM thread pool used for farming (mostly for blocking I/O, but also for some
-    /// compute-intensive operations during proving), defaults to number of logical CPUs
-    /// available on UMA system and number of logical CPUs in first NUMA node on NUMA system, but
-    /// not more than 32 threads
+    /// compute-intensive operations during proving). Defaults to the number of logical CPUs
+    /// on UMA systems, or the number of logical CPUs in the first NUMA node on NUMA systems, but
+    /// limited to 32 threads.
     #[arg(long)]
     farming_thread_pool_size: Option<NonZeroUsize>,
     /// Disable farm locking, for example if file system doesn't support it