Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 9 additions & 8 deletions vortex-array/src/arrays/varbinview/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,13 @@ impl VarBinViewArray {
dtype: DType,
validity: Validity,
) -> VortexResult<Self> {
let views_nbytes = views.len();
vortex_ensure!(
views_nbytes.is_multiple_of(size_of::<BinaryView>()),
"Expected views buffer length ({views_nbytes}) to be a multiple of {}",
size_of::<BinaryView>()
Comment on lines +168 to +171
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice

);

// TODO(aduffy): device validation.
if let Some(host) = views.as_host_opt() {
vortex_ensure!(
Expand Down Expand Up @@ -222,14 +229,8 @@ impl VarBinViewArray {
.collect();

let handles = Arc::from(handles);

Self {
dtype,
buffers: handles,
views: BufferHandle::new_host(views.into_byte_buffer()),
validity,
stats_set: Default::default(),
}
let view_handle = BufferHandle::new_host(views.into_byte_buffer());
unsafe { Self::new_handle_unchecked(view_handle, handles, dtype, validity) }
}

/// Construct a new array from `BufferHandle`s without validation.
Expand Down
50 changes: 33 additions & 17 deletions vortex-array/src/arrays/varbinview/vtable/mod.rs
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use std::mem::size_of;
use std::sync::Arc;

use kernel::PARENT_KERNELS;
use vortex_buffer::Buffer;
use vortex_buffer::ByteBuffer;
use vortex_dtype::DType;
use vortex_error::VortexExpect;
use vortex_error::VortexResult;
use vortex_error::vortex_bail;
use vortex_error::vortex_err;
Expand Down Expand Up @@ -80,20 +79,9 @@ impl VTable for VarBinViewVTable {
buffers: &[BufferHandle],
children: &dyn ArrayChildren,
) -> VortexResult<VarBinViewArray> {
if buffers.is_empty() {
vortex_bail!("Expected at least 1 buffer, got {}", buffers.len());
}
let mut buffers: Vec<ByteBuffer> = buffers
.iter()
.map(|b| b.clone().try_to_host_sync())
.collect::<VortexResult<Vec<_>>>()?;
let views = buffers.pop().vortex_expect("buffers non-empty");

let views = Buffer::<BinaryView>::from_byte_buffer(views);

if views.len() != len {
vortex_bail!("Expected {} views, got {}", len, views.len());
}
let Some((views_handle, data_handles)) = buffers.split_last() else {
vortex_bail!("Expected at least 1 buffer, got 0");
};

let validity = if children.is_empty() {
Validity::from(dtype.nullability())
Expand All @@ -104,7 +92,35 @@ impl VTable for VarBinViewVTable {
vortex_bail!("Expected 0 or 1 children, got {}", children.len());
};

VarBinViewArray::try_new(views, Arc::from(buffers), dtype.clone(), validity)
let views_nbytes = views_handle.len();
let expected_views_nbytes = len
.checked_mul(size_of::<BinaryView>())
.ok_or_else(|| vortex_err!("views byte length overflow for len={len}"))?;
if views_nbytes != expected_views_nbytes {
vortex_bail!(
"Expected views buffer length {} bytes, got {} bytes",
expected_views_nbytes,
views_nbytes
);
}

// If any buffer is on device, skip host validation and use try_new_handle.
if buffers.iter().any(|b| b.is_on_device()) {
return VarBinViewArray::try_new_handle(
views_handle.clone(),
Arc::from(data_handles.to_vec()),
dtype.clone(),
validity,
);
}
Comment on lines +108 to +115
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we not always use this path?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

try_new does validation reading buffer contents


let data_buffers = data_handles
.iter()
.map(|b| b.as_host().clone())
.collect::<Vec<_>>();
let views = Buffer::<BinaryView>::from_byte_buffer(views_handle.clone().as_host().clone());

VarBinViewArray::try_new(views, Arc::from(data_buffers), dtype.clone(), validity)
}

fn with_children(array: &mut Self::Array, children: Vec<ArrayRef>) -> VortexResult<()> {
Expand Down
Loading