From 4570ba7fac4db2d4f75259c1ff146871ff30592a Mon Sep 17 00:00:00 2001 From: coastalwhite Date: Fri, 19 Jul 2024 13:00:51 +0200 Subject: [PATCH] perf: use mmap-ed memory if possible in Parquet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This seems to have negative performance without prefetching or with madvise. The current implementation uses prefetching to the L2 cache. This seems to have ~5% increased performance for multithreaded and ~10% increased performance on single-threaded. All this testing is done on cold file reads. Warm file reads seems to be faster as well, but it is more noisy. Multi-threaded: ``` Benchmark 1: ./plparbench-before Time (mean ± σ): 6.049 s ± 0.031 s [User: 5.813 s, System: 5.811 s] Range (min … max): 6.013 s … 6.086 s 5 runs Benchmark 2: ./plparbench-after Time (mean ± σ): 5.761 s ± 0.020 s [User: 5.083 s, System: 5.792 s] Range (min … max): 5.735 s … 5.788 s 5 runs Summary ./plparbench-after ran 1.05 ± 0.01 times faster than ./plparbench-before ``` Single-threaded: ``` Benchmark 1: ./plparbench-before Time (mean ± σ): 13.601 s ± 0.184 s [User: 5.295 s, System: 5.206 s] Range (min … max): 13.447 s … 13.858 s 5 runs Benchmark 2: ./plparbench-after Time (mean ± σ): 12.398 s ± 0.152 s [User: 4.862 s, System: 5.134 s] Range (min … max): 12.276 s … 12.664 s 5 runs Summary ./plparbench-after ran 1.10 ± 0.02 times faster than ./plparbench-before ``` --- Cargo.lock | 1 + Cargo.toml | 1 + crates/polars-io/Cargo.toml | 4 +- crates/polars-io/src/mmap.rs | 12 ++ crates/polars-io/src/parquet/read/mmap.rs | 21 ++- .../polars-io/src/parquet/read/read_impl.rs | 10 +- crates/polars-parquet/Cargo.toml | 2 +- crates/polars-parquet/src/parquet/read/mod.rs | 4 +- .../src/parquet/read/page/indexed_reader.rs | 5 +- .../src/parquet/read/page/memreader.rs | 156 ++++++++++++++---- .../src/parquet/read/page/mod.rs | 2 +- .../src/parquet/read/page/reader.rs | 6 +- .../src/parquet/read/page/stream.rs | 4 +- crates/polars-utils/Cargo.toml | 2 + crates/polars-utils/src/lib.rs | 2 + crates/polars-utils/src/mmap.rs | 73 ++++++++ 16 files changed, 245 insertions(+), 60 deletions(-) create mode 100644 crates/polars-utils/src/mmap.rs diff --git a/Cargo.lock b/Cargo.lock index a8ffdd804c23f..3d81487a0aadd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3416,6 +3416,7 @@ dependencies = [ "bytemuck", "hashbrown", "indexmap", + "memmap2", "num-traits", "once_cell", "polars-error", diff --git a/Cargo.toml b/Cargo.toml index b996a7877bfe4..2efb3e722d491 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ indexmap = { version = "2", features = ["std"] } itoa = "1.0.6" itoap = { version = "1", features = ["simd"] } memchr = "2.6" +memmap = { package = "memmap2", version = "0.7" } multiversion = "0.7" ndarray = { version = "0.15", default-features = false } num-traits = "0.2" diff --git a/crates/polars-io/Cargo.toml b/crates/polars-io/Cargo.toml index c99cc6e407f97..aa2dc674f7a9f 100644 --- a/crates/polars-io/Cargo.toml +++ b/crates/polars-io/Cargo.toml @@ -14,7 +14,7 @@ polars-error = { workspace = true } polars-json = { workspace = true, optional = true } polars-parquet = { workspace = true, optional = true } polars-time = { workspace = true, features = [], optional = true } -polars-utils = { workspace = true } +polars-utils = { workspace = true, features = ['mmap'] } ahash = { workspace = true } arrow = { workspace = true } @@ -30,7 +30,7 @@ futures = { workspace = true, optional = true } glob = { version = "0.3" } itoa = { workspace = true, optional = true } memchr = { workspace = true } -memmap = { package = "memmap2", version = "0.7" } +memmap = { workspace = true } num-traits = { workspace = true } object_store = { workspace = true, optional = true } once_cell = { workspace = true } diff --git a/crates/polars-io/src/mmap.rs b/crates/polars-io/src/mmap.rs index 5f454ba081c64..fe5d088eeb23e 100644 --- a/crates/polars-io/src/mmap.rs +++ b/crates/polars-io/src/mmap.rs @@ -8,6 +8,8 @@ use memmap::Mmap; use once_cell::sync::Lazy; use polars_core::config::verbose; use polars_error::{polars_bail, PolarsResult}; +use polars_parquet::parquet::read::MemSlice; +use polars_utils::mmap::MmapSlice; // Keep track of memory mapped files so we don't write to them while reading // Use a btree as it uses less memory than a hashmap and this thing never shrinks. @@ -143,6 +145,16 @@ impl std::ops::Deref for ReaderBytes<'_> { } } +impl<'a> ReaderBytes<'a> { + pub fn into_mem_slice(self) -> MemSlice { + match self { + ReaderBytes::Borrowed(v) => MemSlice::from_slice(v), + ReaderBytes::Owned(v) => MemSlice::from_vec(v), + ReaderBytes::Mapped(v, _) => MemSlice::from_mmap(MmapSlice::new(v)), + } + } +} + impl<'a, T: 'a + MmapBytesReader> From<&'a mut T> for ReaderBytes<'a> { fn from(m: &'a mut T) -> Self { match m.to_bytes() { diff --git a/crates/polars-io/src/parquet/read/mmap.rs b/crates/polars-io/src/parquet/read/mmap.rs index 8ac959069f76a..8b7bad655782d 100644 --- a/crates/polars-io/src/parquet/read/mmap.rs +++ b/crates/polars-io/src/parquet/read/mmap.rs @@ -4,7 +4,7 @@ use bytes::Bytes; #[cfg(feature = "async")] use polars_core::datatypes::PlHashMap; use polars_error::PolarsResult; -use polars_parquet::parquet::read::MemReader; +use polars_parquet::parquet::read::{MemReader, MemSlice}; use polars_parquet::read::{ column_iter_to_arrays, get_field_columns, ArrayIter, BasicDecompressor, ColumnChunkMetaData, PageReader, @@ -21,8 +21,8 @@ use polars_parquet::read::{ /// b. asynchronously fetch them in parallel, for example using object_store /// c. store the data in this data structure /// d. when all the data is available deserialize on multiple threads, for example using rayon -pub enum ColumnStore<'a> { - Local(&'a [u8]), +pub enum ColumnStore { + Local(MemSlice), #[cfg(feature = "async")] Fetched(PlHashMap), } @@ -33,7 +33,7 @@ pub(super) fn mmap_columns<'a>( store: &'a ColumnStore, columns: &'a [ColumnChunkMetaData], field_name: &str, -) -> Vec<(&'a ColumnChunkMetaData, &'a [u8])> { +) -> Vec<(&'a ColumnChunkMetaData, MemSlice)> { get_field_columns(columns, field_name) .into_iter() .map(|meta| _mmap_single_column(store, meta)) @@ -43,10 +43,10 @@ pub(super) fn mmap_columns<'a>( fn _mmap_single_column<'a>( store: &'a ColumnStore, meta: &'a ColumnChunkMetaData, -) -> (&'a ColumnChunkMetaData, &'a [u8]) { +) -> (&'a ColumnChunkMetaData, MemSlice) { let (start, len) = meta.byte_range(); let chunk = match store { - ColumnStore::Local(file) => &file[start as usize..(start + len) as usize], + ColumnStore::Local(mem_slice) => mem_slice.slice(start as usize, (start + len) as usize), #[cfg(all(feature = "async", feature = "parquet"))] ColumnStore::Fetched(fetched) => { let entry = fetched.get(&start).unwrap_or_else(|| { @@ -54,7 +54,7 @@ fn _mmap_single_column<'a>( "mmap_columns: column with start {start} must be prefetched in ColumnStore.\n" ) }); - entry.as_ref() + MemSlice::from_slice(entry.as_ref()) }, }; (meta, chunk) @@ -63,7 +63,7 @@ fn _mmap_single_column<'a>( // similar to arrow2 serializer, except this accepts a slice instead of a vec. // this allows us to memory map pub(super) fn to_deserializer<'a>( - columns: Vec<(&ColumnChunkMetaData, &'a [u8])>, + columns: Vec<(&ColumnChunkMetaData, MemSlice)>, field: Field, num_rows: usize, chunk_size: Option, @@ -73,8 +73,11 @@ pub(super) fn to_deserializer<'a>( let (columns, types): (Vec<_>, Vec<_>) = columns .into_iter() .map(|(column_meta, chunk)| { + // Advise fetching the data for the column chunk + chunk.populate(); + let pages = PageReader::new( - MemReader::from_slice(chunk), + MemReader::new(chunk), column_meta, std::sync::Arc::new(|_, _| true), vec![], diff --git a/crates/polars-io/src/parquet/read/read_impl.rs b/crates/polars-io/src/parquet/read/read_impl.rs index 97d2a93a9cec6..7b3d82fe2f704 100644 --- a/crates/polars-io/src/parquet/read/read_impl.rs +++ b/crates/polars-io/src/parquet/read/read_impl.rs @@ -7,6 +7,7 @@ use arrow::datatypes::ArrowSchemaRef; use polars_core::prelude::*; use polars_core::utils::{accumulate_dataframes_vertical, split_df}; use polars_core::POOL; +use polars_parquet::parquet::read::MemSlice; use polars_parquet::read::{self, ArrayIter, FileMetaData, PhysicalType, RowGroupMetaData}; use rayon::prelude::*; @@ -446,8 +447,7 @@ pub fn read_parquet( } let reader = ReaderBytes::from(&mut reader); - let bytes = reader.deref(); - let store = mmap::ColumnStore::Local(bytes); + let store = mmap::ColumnStore::Local(reader.into_mem_slice()); let dfs = rg_to_dfs( &store, @@ -492,8 +492,12 @@ impl FetchRowGroupsFromMmapReader { let reader_bytes = get_reader_bytes(reader_ptr)?; Ok(FetchRowGroupsFromMmapReader(reader_bytes)) } + fn fetch_row_groups(&mut self, _row_groups: Range) -> PolarsResult { - Ok(mmap::ColumnStore::Local(self.0.deref())) + // @TODO: we can something smarter here with mmap + Ok(mmap::ColumnStore::Local(MemSlice::from_slice( + self.0.deref(), + ))) } } diff --git a/crates/polars-parquet/Cargo.toml b/crates/polars-parquet/Cargo.toml index 2e8418c5e03d4..5c62479ccaa30 100644 --- a/crates/polars-parquet/Cargo.toml +++ b/crates/polars-parquet/Cargo.toml @@ -23,7 +23,7 @@ futures = { workspace = true, optional = true } num-traits = { workspace = true } polars-compute = { workspace = true } polars-error = { workspace = true } -polars-utils = { workspace = true } +polars-utils = { workspace = true, features = ["mmap"] } simdutf8 = { workspace = true } parquet-format-safe = "0.2" diff --git a/crates/polars-parquet/src/parquet/read/mod.rs b/crates/polars-parquet/src/parquet/read/mod.rs index 680ed676194b0..7b22cd4604138 100644 --- a/crates/polars-parquet/src/parquet/read/mod.rs +++ b/crates/polars-parquet/src/parquet/read/mod.rs @@ -17,8 +17,8 @@ pub use metadata::{deserialize_metadata, read_metadata, read_metadata_with_size} #[cfg(feature = "async")] pub use page::{get_page_stream, get_page_stream_from_column_start}; pub use page::{ - CowBuffer, IndexedPageReader, MemReader, MemReaderSlice, PageFilter, PageIterator, - PageMetaData, PageReader, + CowBuffer, IndexedPageReader, MemReader, MemSlice, PageFilter, PageIterator, PageMetaData, + PageReader, }; #[cfg(feature = "async")] pub use stream::read_metadata as read_metadata_async; diff --git a/crates/polars-parquet/src/parquet/read/page/indexed_reader.rs b/crates/polars-parquet/src/parquet/read/page/indexed_reader.rs index 7f8bc34764c47..4e620202c6382 100644 --- a/crates/polars-parquet/src/parquet/read/page/indexed_reader.rs +++ b/crates/polars-parquet/src/parquet/read/page/indexed_reader.rs @@ -1,9 +1,8 @@ use std::collections::VecDeque; use std::io::{Seek, SeekFrom}; -use super::memreader::MemReader; +use super::memreader::{MemReader, MemSlice}; use super::reader::{finish_page, read_page_header, PageMetaData}; -use super::MemReaderSlice; use crate::parquet::error::ParquetError; use crate::parquet::indexes::{FilteredPage, Interval}; use crate::parquet::metadata::{ColumnChunkMetaData, Descriptor}; @@ -44,7 +43,7 @@ fn read_page( reader: &mut MemReader, start: u64, length: usize, -) -> Result<(ParquetPageHeader, MemReaderSlice), ParquetError> { +) -> Result<(ParquetPageHeader, MemSlice), ParquetError> { // seek to the page reader.seek(SeekFrom::Start(start))?; diff --git a/crates/polars-parquet/src/parquet/read/page/memreader.rs b/crates/polars-parquet/src/parquet/read/page/memreader.rs index 4c200e506474d..bf2ac87dba0f2 100644 --- a/crates/polars-parquet/src/parquet/read/page/memreader.rs +++ b/crates/polars-parquet/src/parquet/read/page/memreader.rs @@ -1,12 +1,27 @@ -use std::io; use std::ops::Deref; use std::sync::Arc; +use std::{fmt, io}; + +use polars_utils::mmap::MmapSlice; + +#[derive(Clone)] +enum MemSliceInner { + Mmap(MmapSlice), + Allocated(AllocatedSlice), +} + +#[derive(Clone)] +pub struct AllocatedSlice { + data: Arc<[u8]>, + start: usize, + end: usize, +} /// A cursor over a segment of heap allocated memory. This is used for the Parquet reader to avoid /// sequential allocations. #[derive(Debug, Clone)] pub struct MemReader { - data: Arc<[u8]>, + data: MemSlice, position: usize, } @@ -14,14 +29,16 @@ pub struct MemReader { /// /// This should not outlast the original the original [`MemReader`] because it still owns all the /// memory. -#[derive(Debug, Clone)] -pub struct MemReaderSlice { - data: Arc<[u8]>, - start: usize, - end: usize, +#[derive(Clone)] +pub struct MemSlice(MemSliceInner); + +impl fmt::Debug for MemSlice { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_tuple("MemSlice").field(&self.deref()).finish() + } } -impl Default for MemReaderSlice { +impl Default for AllocatedSlice { fn default() -> Self { let slice: &[u8] = &[]; Self { @@ -32,18 +49,27 @@ impl Default for MemReaderSlice { } } -impl Deref for MemReaderSlice { +impl Default for MemSlice { + fn default() -> Self { + Self(MemSliceInner::Allocated(AllocatedSlice::default())) + } +} + +impl Deref for MemSlice { type Target = [u8]; #[inline(always)] fn deref(&self) -> &Self::Target { - &self.data[self.start..self.end] + match &self.0 { + MemSliceInner::Mmap(v) => v.deref(), + MemSliceInner::Allocated(v) => &v.data[v.start..v.end], + } } } #[derive(Debug, Clone)] pub enum CowBuffer { - Borrowed(MemReaderSlice), + Borrowed(MemSlice), Owned(Vec), } @@ -60,8 +86,7 @@ impl Deref for CowBuffer { } impl MemReader { - #[inline(always)] - pub fn new(data: Arc<[u8]>) -> Self { + pub fn new(data: MemSlice) -> Self { Self { data, position: 0 } } @@ -82,14 +107,12 @@ impl MemReader { #[inline(always)] pub fn from_slice(data: &[u8]) -> Self { - let data = data.into(); - Self { data, position: 0 } + Self::new(MemSlice::from_slice(data)) } #[inline(always)] pub fn from_vec(data: Vec) -> Self { - let data = data.into_boxed_slice().into(); - Self { data, position: 0 } + Self::new(MemSlice::from_vec(data)) } #[inline(always)] @@ -100,17 +123,11 @@ impl MemReader { } #[inline(always)] - pub fn read_slice(&mut self, n: usize) -> MemReaderSlice { + pub fn read_slice(&mut self, n: usize) -> MemSlice { let start = self.position; let end = usize::min(self.position + n, self.data.len()); - self.position = end; - - MemReaderSlice { - data: self.data.clone(), - start, - end, - } + self.data.slice(start, end) } } @@ -149,33 +166,104 @@ impl io::Seek for MemReader { }, }; - eprintln!( - "pos = {}, new_pos = {}, seek = {:?}", - self.position, position, pos - ); - self.position = position; Ok(position as u64) } } -impl MemReaderSlice { +impl MemSlice { #[inline(always)] pub fn to_vec(self) -> Vec { - <[u8]>::to_vec(&self) + <[u8]>::to_vec(self.deref()) } #[inline] pub fn from_vec(v: Vec) -> Self { let end = v.len(); - Self { - data: v.into(), + Self(MemSliceInner::Allocated(AllocatedSlice { + data: v.into_boxed_slice().into(), start: 0, end, + })) + } + + #[inline] + pub fn from_slice(slice: &[u8]) -> Self { + let end = slice.len(); + Self(MemSliceInner::Allocated(AllocatedSlice { + data: slice.into(), + start: 0, + end, + })) + } + + #[inline] + pub fn populate(&self) { + #[cfg(target_arch = "x86_64")] + { + if self.len() == 0 { + return; + } + + const PREFETCH: unsafe fn(*const u8) = { + #[cfg(target_arch = "x86_64")] + { + |ptr| unsafe { + use std::arch::x86_64::*; + _mm_prefetch(ptr as *const _, _MM_HINT_T1); + } + } + + #[cfg(target_arch = "aarch64")] + { + |ptr| unsafe { + use std::arch::aarch64::*; + _prefetch(ptr as *const _, _PREFETCH_READ, _PREFETCH_LOCALITY2); + } + } + + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] + { + |_ptr| {} + } + }; + + const PAGE_SIZE: usize = 4096; + + for i in 0..self.len() / PAGE_SIZE { + unsafe { PREFETCH(self[i * PAGE_SIZE..].as_ptr()) } + } + unsafe { PREFETCH(self[self.len() - 1..].as_ptr()) } } } + + #[inline] + pub fn from_mmap(mmap: MmapSlice) -> Self { + Self(MemSliceInner::Mmap(mmap)) + } + + #[inline] + #[track_caller] + pub fn slice(&self, start: usize, end: usize) -> Self { + Self(match &self.0 { + MemSliceInner::Mmap(v) => MemSliceInner::Mmap(v.slice(start, end)), + MemSliceInner::Allocated(v) => MemSliceInner::Allocated({ + let len = v.end - v.start; + + assert!(start <= end); + assert!(start <= len); + assert!(end <= len); + + AllocatedSlice { + data: v.data.clone(), + start: v.start + start, + end: v.start + end, + } + }), + }) + } } impl CowBuffer { diff --git a/crates/polars-parquet/src/parquet/read/page/mod.rs b/crates/polars-parquet/src/parquet/read/page/mod.rs index 350e49257e799..0349839efbf82 100644 --- a/crates/polars-parquet/src/parquet/read/page/mod.rs +++ b/crates/polars-parquet/src/parquet/read/page/mod.rs @@ -5,7 +5,7 @@ mod reader; mod stream; pub use indexed_reader::IndexedPageReader; -pub use memreader::{CowBuffer, MemReader, MemReaderSlice}; +pub use memreader::{CowBuffer, MemReader, MemSlice}; pub use reader::{PageFilter, PageMetaData, PageReader}; use crate::parquet::error::ParquetError; diff --git a/crates/polars-parquet/src/parquet/read/page/reader.rs b/crates/polars-parquet/src/parquet/read/page/reader.rs index 82b542ee1eb1f..be4821ead78f4 100644 --- a/crates/polars-parquet/src/parquet/read/page/reader.rs +++ b/crates/polars-parquet/src/parquet/read/page/reader.rs @@ -2,8 +2,8 @@ use std::sync::{Arc, OnceLock}; use parquet_format_safe::thrift::protocol::TCompactInputProtocol; -use super::memreader::MemReader; -use super::{MemReaderSlice, PageIterator}; +use super::memreader::{MemReader, MemSlice}; +use super::PageIterator; use crate::parquet::compression::Compression; use crate::parquet::error::{ParquetError, ParquetResult}; use crate::parquet::indexes::Interval; @@ -211,7 +211,7 @@ pub(super) fn build_page(reader: &mut PageReader) -> ParquetResult>, diff --git a/crates/polars-parquet/src/parquet/read/page/stream.rs b/crates/polars-parquet/src/parquet/read/page/stream.rs index ee4cc4f4cd2bd..d2811bf56d440 100644 --- a/crates/polars-parquet/src/parquet/read/page/stream.rs +++ b/crates/polars-parquet/src/parquet/read/page/stream.rs @@ -11,7 +11,7 @@ use crate::parquet::compression::Compression; use crate::parquet::error::{ParquetError, ParquetResult}; use crate::parquet::metadata::{ColumnChunkMetaData, Descriptor}; use crate::parquet::page::{CompressedPage, ParquetPageHeader}; -use crate::parquet::read::MemReaderSlice; +use crate::parquet::read::MemSlice; /// Returns a stream of compressed data pages pub async fn get_page_stream<'a, RR: AsyncRead + Unpin + Send + AsyncSeek>( @@ -119,7 +119,7 @@ fn _get_page_stream( yield finish_page( page_header, - MemReaderSlice::from_vec(std::mem::take(&mut scratch)), + MemSlice::from_vec(std::mem::take(&mut scratch)), compression, &descriptor, None, diff --git a/crates/polars-utils/Cargo.toml b/crates/polars-utils/Cargo.toml index c237d8d56a29a..6eec2021a15f8 100644 --- a/crates/polars-utils/Cargo.toml +++ b/crates/polars-utils/Cargo.toml @@ -15,6 +15,7 @@ ahash = { workspace = true } bytemuck = { workspace = true } hashbrown = { workspace = true } indexmap = { workspace = true } +memmap = { workspace = true, optional = true } num-traits = { workspace = true } once_cell = { workspace = true } raw-cpuid = { workspace = true } @@ -30,5 +31,6 @@ rand = { workspace = true } version_check = { workspace = true } [features] +mmap = ["memmap"] bigidx = [] nightly = [] diff --git a/crates/polars-utils/src/lib.rs b/crates/polars-utils/src/lib.rs index 9ae809f774588..d01c87c77e28d 100644 --- a/crates/polars-utils/src/lib.rs +++ b/crates/polars-utils/src/lib.rs @@ -38,6 +38,8 @@ pub mod wasm; pub mod float; pub mod index; pub mod io; +#[cfg(feature = "mmap")] +pub mod mmap; pub mod nulls; pub mod ord; pub mod partitioned; diff --git a/crates/polars-utils/src/mmap.rs b/crates/polars-utils/src/mmap.rs new file mode 100644 index 0000000000000..b89cf25848473 --- /dev/null +++ b/crates/polars-utils/src/mmap.rs @@ -0,0 +1,73 @@ +use core::fmt; +use std::ops::Deref; +use std::sync::Arc; + +pub use memmap::Mmap; + +pub struct MmapSlice { + // We keep the Mmap around to ensure it is still valid. + mmap: Arc, + ptr: *const u8, + len: usize, +} + +// SAFETY: This structure is read-only and does not contain any non-sync or non-send data. +unsafe impl Sync for MmapSlice {} +unsafe impl Send for MmapSlice {} + +impl Deref for MmapSlice { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + // SAFETY: Invariant of MmapSlice + unsafe { std::slice::from_raw_parts(self.ptr, self.len) } + } +} + +impl fmt::Debug for MmapSlice { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt.debug_tuple("MmapSlice").field(&self.deref()).finish() + } +} + +impl Clone for MmapSlice { + fn clone(&self) -> Self { + Self { + mmap: self.mmap.clone(), + ptr: self.ptr, + len: self.len, + } + } +} + +impl MmapSlice { + #[inline] + pub fn new(mmap: Mmap) -> Self { + let slice: &[u8] = &mmap; + + let ptr = slice as *const [u8] as *const u8; + let len = slice.len(); + + let mmap = Arc::new(mmap); + + Self { mmap, ptr, len } + } + + #[inline] + #[track_caller] + pub fn slice(&self, start: usize, end: usize) -> Self { + assert!(start <= end); + assert!(start <= self.len()); + assert!(end <= self.len()); + + // SAFETY: Start and end are within the slice + let ptr = unsafe { self.ptr.add(start) }; + let len = end - start; + + Self { + mmap: self.mmap.clone(), + ptr, + len, + } + } +}