Merge pull request #1642 from subspace/farmer-refactoring-and-cleanups

Farmer refactoring and cleanups
autonomys · Jul 6, 2023 · ba0ee8b · ba0ee8b
2 parents 6e7f8c7 + 1262a79
commit ba0ee8b
Show file tree

Hide file tree

Showing 19 changed files with 192 additions and 311 deletions.
diff --git a/crates/pallet-subspace/src/mock.rs b/crates/pallet-subspace/src/mock.rs
@@ -410,13 +410,12 @@ pub fn create_signed_vote(
     let pieces_in_sector = farmer_protocol_info.max_pieces_in_sector;
     let sector_size = sector_size(pieces_in_sector);
 
-    for (sector_offset, sector_index) in iter::from_fn(|| Some(rand::random())).enumerate() {
+    for sector_index in iter::from_fn(|| Some(rand::random())) {
         let mut plotted_sector_bytes = vec![0; sector_size];
         let mut plotted_sector_metadata_bytes = vec![0; SectorMetadata::encoded_size()];
 
         let plotted_sector = block_on(plot_sector::<_, PosTable>(
             &public_key,
-            sector_offset,
             sector_index,
             archived_history_segment,
             PieceGetterRetryPolicy::default(),
@@ -426,7 +425,6 @@ pub fn create_signed_vote(
             pieces_in_sector,
             &mut plotted_sector_bytes,
             &mut plotted_sector_metadata_bytes,
-            Default::default(),
         ))
         .unwrap();
 

diff --git a/crates/sp-lightclient/src/tests.rs b/crates/sp-lightclient/src/tests.rs
@@ -153,13 +153,12 @@ fn valid_header(
     let pieces_in_sector = farmer_parameters.farmer_protocol_info.max_pieces_in_sector;
     let sector_size = sector_size(pieces_in_sector);
 
-    for (sector_offset, sector_index) in iter::from_fn(|| Some(rand::random())).enumerate() {
+    for sector_index in iter::from_fn(|| Some(rand::random())) {
         let mut plotted_sector_bytes = vec![0; sector_size];
         let mut plotted_sector_metadata_bytes = vec![0; SectorMetadata::encoded_size()];
 
         let plotted_sector = block_on(plot_sector::<_, PosTable>(
             &public_key,
-            sector_offset,
             sector_index,
             &farmer_parameters.archived_segment.pieces,
             PieceGetterRetryPolicy::default(),
@@ -169,7 +168,6 @@ fn valid_header(
             pieces_in_sector,
             &mut plotted_sector_bytes,
             &mut plotted_sector_metadata_bytes,
-            Default::default(),
         ))
         .unwrap();
 

diff --git a/crates/subspace-core-primitives/src/lib.rs b/crates/subspace-core-primitives/src/lib.rs
@@ -489,7 +489,7 @@ impl SegmentHeader {
 }
 
 /// Sector index in consensus
-pub type SectorIndex = u64;
+pub type SectorIndex = u16;
 
 // TODO: Versioned solution enum
 /// Farmer solution for slot challenge.

diff --git a/crates/subspace-farmer-components/benches/auditing.rs b/crates/subspace-farmer-components/benches/auditing.rs
@@ -11,8 +11,8 @@ use subspace_archiving::archiver::Archiver;
 use subspace_core_primitives::crypto::kzg;
 use subspace_core_primitives::crypto::kzg::Kzg;
 use subspace_core_primitives::{
-    Blake2b256Hash, HistorySize, PublicKey, Record, RecordedHistorySegment, SectorId, SegmentIndex,
-    SolutionRange,
+    Blake2b256Hash, HistorySize, PublicKey, Record, RecordedHistorySegment, SectorId, SectorIndex,
+    SegmentIndex, SolutionRange,
 };
 use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::auditing::audit_sector;
@@ -42,7 +42,6 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         .unwrap_or(10);
 
     let public_key = PublicKey::default();
-    let sector_offset = 0;
     let sector_index = 0;
     let mut input = RecordedHistorySegment::new_boxed();
     StdRng::seed_from_u64(42).fill(AsMut::<[u8]>::as_mut(input.as_mut()));
@@ -116,7 +115,6 @@ pub fn criterion_benchmark(c: &mut Criterion) {
 
         let plotted_sector = block_on(plot_sector::<_, PosTable>(
             &public_key,
-            sector_offset,
             sector_index,
             &archived_history_segment,
             PieceGetterRetryPolicy::default(),
@@ -126,7 +124,6 @@ pub fn criterion_benchmark(c: &mut Criterion) {
             pieces_in_sector,
             &mut plotted_sector_bytes,
             &mut plotted_sector_metadata_bytes,
-            Default::default(),
         ))
         .unwrap();
 
@@ -196,7 +193,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
                     for (sector_index, sector) in plot_mmap
                         .chunks_exact(sector_size)
                         .enumerate()
-                        .map(|(sector_index, sector)| (sector_index as u64, sector))
+                        .map(|(sector_index, sector)| (sector_index as SectorIndex, sector))
                     {
                         audit_sector(
                             black_box(&public_key),

diff --git a/crates/subspace-farmer-components/benches/plotting.rs b/crates/subspace-farmer-components/benches/plotting.rs
@@ -26,7 +26,6 @@ fn criterion_benchmark(c: &mut Criterion) {
         .unwrap_or_else(|_error| MAX_PIECES_IN_SECTOR);
 
     let public_key = PublicKey::default();
-    let sector_offset = 0;
     let sector_index = 0;
     let mut input = RecordedHistorySegment::new_boxed();
     StdRng::seed_from_u64(42).fill(AsMut::<[u8]>::as_mut(input.as_mut()));
@@ -67,7 +66,6 @@ fn criterion_benchmark(c: &mut Criterion) {
         b.iter(|| {
             block_on(plot_sector::<_, PosTable>(
                 black_box(&public_key),
-                black_box(sector_offset),
                 black_box(sector_index),
                 black_box(&archived_history_segment),
                 black_box(PieceGetterRetryPolicy::default()),
@@ -77,7 +75,6 @@ fn criterion_benchmark(c: &mut Criterion) {
                 black_box(pieces_in_sector),
                 black_box(&mut sector_bytes),
                 black_box(&mut sector_metadata_bytes),
-                Default::default(),
             ))
             .unwrap();
         })

diff --git a/crates/subspace-farmer-components/benches/proving.rs b/crates/subspace-farmer-components/benches/proving.rs
@@ -44,7 +44,6 @@ pub fn criterion_benchmark(c: &mut Criterion) {
 
     let keypair = Keypair::from_bytes(&[0; 96]).unwrap();
     let public_key = PublicKey::from(keypair.public.to_bytes());
-    let sector_offset = 0;
     let sector_index = 0;
     let mut input = RecordedHistorySegment::new_boxed();
     let mut rng = StdRng::seed_from_u64(42);
@@ -119,7 +118,6 @@ pub fn criterion_benchmark(c: &mut Criterion) {
 
         let plotted_sector = block_on(plot_sector::<_, PosTable>(
             &public_key,
-            sector_offset,
             sector_index,
             &archived_history_segment,
             PieceGetterRetryPolicy::default(),
@@ -129,7 +127,6 @@ pub fn criterion_benchmark(c: &mut Criterion) {
             pieces_in_sector,
             &mut plotted_sector_bytes,
             &mut plotted_sector_metadata_bytes,
-            Default::default(),
         ))
         .unwrap();
 

diff --git a/crates/subspace-farmer-components/benches/reading.rs b/crates/subspace-farmer-components/benches/reading.rs
@@ -41,7 +41,6 @@ pub fn criterion_benchmark(c: &mut Criterion) {
         .unwrap_or(10);
 
     let public_key = PublicKey::default();
-    let sector_offset = 0;
     let sector_index = 0;
     let mut input = RecordedHistorySegment::new_boxed();
     StdRng::seed_from_u64(42).fill(AsMut::<[u8]>::as_mut(input.as_mut()));
@@ -113,7 +112,6 @@ pub fn criterion_benchmark(c: &mut Criterion) {
 
         let plotted_sector = block_on(plot_sector::<_, PosTable>(
             &public_key,
-            sector_offset,
             sector_index,
             &archived_history_segment,
             PieceGetterRetryPolicy::default(),
@@ -123,7 +121,6 @@ pub fn criterion_benchmark(c: &mut Criterion) {
             pieces_in_sector,
             &mut plotted_sector_bytes,
             &mut plotted_sector_metadata_bytes,
-            Default::default(),
         ))
         .unwrap();
 

diff --git a/crates/subspace-farmer-components/src/auditing.rs b/crates/subspace-farmer-components/src/auditing.rs
@@ -3,7 +3,7 @@ use crate::sector::{SectorContentsMap, SectorMetadata};
 use std::collections::VecDeque;
 use std::mem;
 use subspace_core_primitives::crypto::Scalar;
-use subspace_core_primitives::{Blake2b256Hash, PublicKey, SectorId, SolutionRange};
+use subspace_core_primitives::{Blake2b256Hash, PublicKey, SectorId, SectorIndex, SolutionRange};
 use subspace_verification::is_within_solution_range;
 
 #[derive(Debug, Clone)]
@@ -19,7 +19,7 @@ pub(crate) struct ChunkCandidate {
 /// and seek back afterwards if necessary).
 pub fn audit_sector<'a>(
     public_key: &'a PublicKey,
-    sector_index: u64,
+    sector_index: SectorIndex,
     global_challenge: &Blake2b256Hash,
     solution_range: SolutionRange,
     sector: &'a [u8],

diff --git a/crates/subspace-farmer-components/src/lib.rs b/crates/subspace-farmer-components/src/lib.rs
@@ -16,7 +16,6 @@
 
 pub mod auditing;
 pub mod file_ext;
-pub mod piece_caching;
 pub mod plotting;
 pub mod proving;
 pub mod reading;

diff --git a/crates/subspace-farmer-components/src/piece_caching.rs b/crates/subspace-farmer-components/src/piece_caching.rs
diff --git a/crates/subspace-farmer-components/src/plotting.rs b/crates/subspace-farmer-components/src/plotting.rs
@@ -1,4 +1,3 @@
-use crate::piece_caching::PieceMemoryCache;
 use crate::sector::{
     sector_record_chunks_size, sector_size, RawSector, RecordMetadata, SectorContentsMap,
     SectorMetadata,
@@ -161,8 +160,7 @@ pub enum PlottingError {
 #[allow(clippy::too_many_arguments)]
 pub async fn plot_sector<PG, PosTable>(
     public_key: &PublicKey,
-    sector_offset: usize,
-    sector_index: u64,
+    sector_index: SectorIndex,
     piece_getter: &PG,
     piece_getter_retry_policy: PieceGetterRetryPolicy,
     farmer_protocol_info: &FarmerProtocolInfo,
@@ -171,7 +169,6 @@ pub async fn plot_sector<PG, PosTable>(
     pieces_in_sector: u16,
     sector_output: &mut [u8],
     sector_metadata_output: &mut [u8],
-    piece_memory_cache: PieceMemoryCache,
 ) -> Result<PlottedSector, PlottingError>
 where
     PG: PieceGetter,
@@ -224,18 +221,14 @@ where
 
         if let Err(error) = download_sector(
             &mut raw_sector,
-            sector_offset,
-            sector_index,
             piece_getter,
             piece_getter_retry_policy,
             kzg,
             &piece_indexes,
-            piece_memory_cache.clone(),
         )
         .await
         {
             warn!(
-                %sector_offset,
                 %sector_index,
                 %error,
                 "Sector plotting attempt failed, will retry later"
@@ -244,6 +237,8 @@ where
             return Err(BackoffError::transient(error));
         }
 
+        debug!(%sector_index, "Sector downloaded successfully");
+
         Ok(())
     })
     .await?;
@@ -400,16 +395,12 @@ where
     })
 }
 
-#[allow(clippy::too_many_arguments)]
 async fn download_sector<PG: PieceGetter>(
     raw_sector: &mut RawSector,
-    sector_offset: usize,
-    sector_index: u64,
     piece_getter: &PG,
     piece_getter_retry_policy: PieceGetterRetryPolicy,
     kzg: &Kzg,
     piece_indexes: &[PieceIndex],
-    piece_memory_cache: PieceMemoryCache,
 ) -> Result<(), PlottingError> {
     // TODO: Make configurable, likely allowing user to specify RAM usage expectations and inferring
     //  concurrency from there
@@ -420,10 +411,6 @@ async fn download_sector<PG: PieceGetter>(
         .map(|piece_index| async {
             let piece_index = *piece_index;
 
-            if let Some(piece) = piece_memory_cache.get_piece(&piece_index.hash()) {
-                return (piece_index, Ok(Some(piece)));
-            }
-
             let piece_result = piece_getter
                 .get_piece(piece_index, piece_getter_retry_policy)
                 .await;
@@ -469,11 +456,7 @@ async fn download_sector<PG: PieceGetter>(
             commitment: *commitment,
             witness: *witness,
         });
-
-        piece_memory_cache.add_piece(piece_index.hash(), piece);
     }
 
-    debug!(%sector_offset, %sector_index, "Sector downloaded successfully");
-
     Ok(())
 }