From 1262a795659656c1d74a8906cf58b5c05c88c3e7 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Wed, 5 Jul 2023 16:43:53 +0300 Subject: [PATCH] Reduce `SectorIndex` from `u64` to `u16`, which will be still sufficient for 64TiB plot, for larger amount of space more plots will be needed --- crates/subspace-core-primitives/src/lib.rs | 2 +- .../benches/auditing.rs | 6 +-- .../src/auditing.rs | 4 +- .../src/plotting.rs | 2 +- .../subspace-farmer/src/single_disk_plot.rs | 40 ++++++++++++++++--- .../src/single_disk_plot/plotting.rs | 5 ++- 6 files changed, 45 insertions(+), 14 deletions(-) diff --git a/crates/subspace-core-primitives/src/lib.rs b/crates/subspace-core-primitives/src/lib.rs index 75af1349a5..0f431f91c8 100644 --- a/crates/subspace-core-primitives/src/lib.rs +++ b/crates/subspace-core-primitives/src/lib.rs @@ -489,7 +489,7 @@ impl SegmentHeader { } /// Sector index in consensus -pub type SectorIndex = u64; +pub type SectorIndex = u16; // TODO: Versioned solution enum /// Farmer solution for slot challenge. diff --git a/crates/subspace-farmer-components/benches/auditing.rs b/crates/subspace-farmer-components/benches/auditing.rs index a31606b401..c6d5f79fbe 100644 --- a/crates/subspace-farmer-components/benches/auditing.rs +++ b/crates/subspace-farmer-components/benches/auditing.rs @@ -11,8 +11,8 @@ use subspace_archiving::archiver::Archiver; use subspace_core_primitives::crypto::kzg; use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::{ - Blake2b256Hash, HistorySize, PublicKey, Record, RecordedHistorySegment, SectorId, SegmentIndex, - SolutionRange, + Blake2b256Hash, HistorySize, PublicKey, Record, RecordedHistorySegment, SectorId, SectorIndex, + SegmentIndex, SolutionRange, }; use subspace_erasure_coding::ErasureCoding; use subspace_farmer_components::auditing::audit_sector; @@ -193,7 +193,7 @@ pub fn criterion_benchmark(c: &mut Criterion) { for (sector_index, sector) in plot_mmap .chunks_exact(sector_size) .enumerate() - .map(|(sector_index, sector)| (sector_index as u64, sector)) + .map(|(sector_index, sector)| (sector_index as SectorIndex, sector)) { audit_sector( black_box(&public_key), diff --git a/crates/subspace-farmer-components/src/auditing.rs b/crates/subspace-farmer-components/src/auditing.rs index d70199504c..a1a3913beb 100644 --- a/crates/subspace-farmer-components/src/auditing.rs +++ b/crates/subspace-farmer-components/src/auditing.rs @@ -3,7 +3,7 @@ use crate::sector::{SectorContentsMap, SectorMetadata}; use std::collections::VecDeque; use std::mem; use subspace_core_primitives::crypto::Scalar; -use subspace_core_primitives::{Blake2b256Hash, PublicKey, SectorId, SolutionRange}; +use subspace_core_primitives::{Blake2b256Hash, PublicKey, SectorId, SectorIndex, SolutionRange}; use subspace_verification::is_within_solution_range; #[derive(Debug, Clone)] @@ -19,7 +19,7 @@ pub(crate) struct ChunkCandidate { /// and seek back afterwards if necessary). pub fn audit_sector<'a>( public_key: &'a PublicKey, - sector_index: u64, + sector_index: SectorIndex, global_challenge: &Blake2b256Hash, solution_range: SolutionRange, sector: &'a [u8], diff --git a/crates/subspace-farmer-components/src/plotting.rs b/crates/subspace-farmer-components/src/plotting.rs index fe04112bcb..a47865a22a 100644 --- a/crates/subspace-farmer-components/src/plotting.rs +++ b/crates/subspace-farmer-components/src/plotting.rs @@ -160,7 +160,7 @@ pub enum PlottingError { #[allow(clippy::too_many_arguments)] pub async fn plot_sector( public_key: &PublicKey, - sector_index: u64, + sector_index: SectorIndex, piece_getter: &PG, piece_getter_retry_policy: PieceGetterRetryPolicy, farmer_protocol_info: &FarmerProtocolInfo, diff --git a/crates/subspace-farmer/src/single_disk_plot.rs b/crates/subspace-farmer/src/single_disk_plot.rs index a86ee9394d..eed379412c 100644 --- a/crates/subspace-farmer/src/single_disk_plot.rs +++ b/crates/subspace-farmer/src/single_disk_plot.rs @@ -349,6 +349,18 @@ pub enum SingleDiskPlotError { /// Current allocated space allocated_space: u64, }, + /// Plot is too large + #[error( + "Plot is too large: allocated {allocated_sectors} sectors ({allocated_space} bytes), max \ + supported is {max_sectors} ({max_space} bytes). Consider creating multiple smaller plots \ + instead." + )] + PlotTooLarge { + allocated_space: u64, + allocated_sectors: u64, + max_space: u64, + max_sectors: u16, + }, } /// Errors that happen in background tasks @@ -525,8 +537,23 @@ impl SingleDiskPlot { let pieces_in_sector = single_disk_plot_info.pieces_in_sector(); let sector_size = sector_size(max_pieces_in_sector); let sector_metadata_size = SectorMetadata::encoded_size(); - let target_sector_count: SectorIndex = - single_disk_plot_info.allocated_space() / sector_size as u64; + let target_sector_count = single_disk_plot_info.allocated_space() / sector_size as u64; + let target_sector_count = match SectorIndex::try_from(target_sector_count) { + Ok(target_sector_count) if target_sector_count < SectorIndex::MAX => { + target_sector_count + } + _ => { + // We use this for both count and index, hence index must not reach actual `MAX` + // (consensus doesn't care about this, just farmer implementation detail) + let max_sectors = SectorIndex::MAX - 1; + return Err(SingleDiskPlotError::PlotTooLarge { + allocated_space: target_sector_count * sector_size as u64, + allocated_sectors: target_sector_count, + max_space: max_sectors as u64 * sector_size as u64, + max_sectors, + }); + } + }; // TODO: Consider file locking to prevent other apps from modifying itS let mut metadata_file = OpenOptions::new() @@ -543,7 +570,8 @@ impl SingleDiskPlot { }; metadata_file.preallocate( - RESERVED_PLOT_METADATA + sector_metadata_size as u64 * target_sector_count, + RESERVED_PLOT_METADATA + + sector_metadata_size as u64 * u64::from(target_sector_count), )?; metadata_file.write_all_at(metadata_header.encode().as_slice(), 0)?; @@ -577,12 +605,12 @@ impl SingleDiskPlot { let metadata_mmap = unsafe { MmapOptions::new() .offset(RESERVED_PLOT_METADATA) - .len(sector_metadata_size * target_sector_count as usize) + .len(sector_metadata_size * usize::from(target_sector_count)) .map(&metadata_file)? }; let mut sectors_metadata = - Vec::::with_capacity(target_sector_count as usize); + Vec::::with_capacity(usize::from(target_sector_count)); for mut sector_metadata_bytes in metadata_mmap .chunks_exact(sector_metadata_size) @@ -605,7 +633,7 @@ impl SingleDiskPlot { .open(directory.join(Self::PLOT_FILE))?, ); - plot_file.preallocate(sector_size as u64 * target_sector_count)?; + plot_file.preallocate(sector_size as u64 * u64::from(target_sector_count))?; let (error_sender, error_receiver) = oneshot::channel(); let error_sender = Arc::new(Mutex::new(Some(error_sender))); diff --git a/crates/subspace-farmer/src/single_disk_plot/plotting.rs b/crates/subspace-farmer/src/single_disk_plot/plotting.rs index 542ba62fc9..8f4170797a 100644 --- a/crates/subspace-farmer/src/single_disk_plot/plotting.rs +++ b/crates/subspace-farmer/src/single_disk_plot/plotting.rs @@ -82,7 +82,10 @@ where }; let mut sector_metadata = unsafe { MmapOptions::new() - .offset(RESERVED_PLOT_METADATA + (sector_index * sector_metadata_size as u64)) + .offset( + RESERVED_PLOT_METADATA + + (u64::from(sector_index) * sector_metadata_size as u64), + ) .len(sector_metadata_size) .map_mut(&metadata_file)? };