Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ use num_traits::NumCast;
use vortex_array::arrays::ConstantArray;
use vortex_array::compute::{CompareKernel, CompareKernelAdapter, Operator, compare};
use vortex_array::{Array, ArrayRef, register_kernel};
use vortex_dtype::{IntegerPType, Nullability, PType, match_each_integer_ptype};
use vortex_dtype::{IntegerPType, Nullability, PType, ToI256, match_each_integer_ptype};
use vortex_error::{VortexExpect, VortexResult};
use vortex_scalar::{DecimalValue, Scalar, ScalarValue, ToI256, match_each_decimal_value};
use vortex_scalar::{DecimalValue, Scalar, ScalarValue, match_each_decimal_value};

use crate::DecimalBytePartsVTable;
use crate::decimal_byte_parts::compute::compare::Sign::Positive;
Expand Down
5 changes: 2 additions & 3 deletions encodings/sparse/src/canonical.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,12 @@ use vortex_array::vtable::{CanonicalVTable, ValidityHelper};
use vortex_array::{Array, Canonical, ToCanonical};
use vortex_buffer::{BitBuffer, Buffer, BufferString, ByteBuffer, buffer, buffer_mut};
use vortex_dtype::{
DType, DecimalDType, IntegerPType, NativePType, Nullability, StructFields,
DType, DecimalDType, IntegerPType, NativeDecimalType, NativePType, Nullability, StructFields,
match_each_integer_ptype, match_each_native_ptype,
};
use vortex_error::{VortexError, VortexExpect, vortex_panic};
use vortex_scalar::{
DecimalScalar, ListScalar, NativeDecimalType, Scalar, StructScalar,
match_each_decimal_value_type,
DecimalScalar, ListScalar, Scalar, StructScalar, match_each_decimal_value_type,
};

use crate::{SparseArray, SparseVTable};
Expand Down
4 changes: 2 additions & 2 deletions fuzz/src/array/compare.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ use vortex_array::compute::{Operator, scalar_cmp};
use vortex_array::validity::Validity;
use vortex_array::{Array, ArrayRef, IntoArray, ToCanonical};
use vortex_buffer::BitBuffer;
use vortex_dtype::{DType, NativePType, match_each_native_ptype};
use vortex_dtype::{DType, NativeDecimalType, NativePType, match_each_native_ptype};
use vortex_error::{VortexExpect, VortexResult, vortex_err};
use vortex_scalar::{NativeDecimalType, Scalar, match_each_decimal_value_type};
use vortex_scalar::{Scalar, match_each_decimal_value_type};

pub fn compare_canonical_array(
array: &dyn Array,
Expand Down
6 changes: 4 additions & 2 deletions fuzz/src/array/take.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,11 @@ use vortex_array::builders::builder_with_capacity;
use vortex_array::validity::Validity;
use vortex_array::{Array, ArrayRef, IntoArray, ToCanonical};
use vortex_buffer::Buffer;
use vortex_dtype::{DType, DecimalDType, NativePType, Nullability, match_each_native_ptype};
use vortex_dtype::{
DType, DecimalDType, NativeDecimalType, NativePType, Nullability, match_each_native_ptype,
};
use vortex_error::VortexResult;
use vortex_scalar::{NativeDecimalType, match_each_decimal_value_type};
use vortex_scalar::match_each_decimal_value_type;

pub fn take_canonical_array_non_nullable_indices(
array: &dyn Array,
Expand Down
4 changes: 2 additions & 2 deletions vortex-array/src/arrays/chunked/compute/sum.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@

use num_traits::PrimInt;
use vortex_dtype::Nullability::Nullable;
use vortex_dtype::{DType, DecimalDType, NativePType, match_each_native_ptype};
use vortex_dtype::{DType, DecimalDType, NativePType, i256, match_each_native_ptype};
use vortex_error::{VortexResult, vortex_bail, vortex_err};
use vortex_scalar::{DecimalScalar, DecimalValue, FromPrimitiveOrF16, Scalar, i256};
use vortex_scalar::{DecimalScalar, DecimalValue, FromPrimitiveOrF16, Scalar};

use crate::arrays::{ChunkedArray, ChunkedVTable};
use crate::compute::{SumKernel, SumKernelAdapter, sum};
Expand Down
4 changes: 2 additions & 2 deletions vortex-array/src/arrays/constant/compute/sum.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use num_traits::{CheckedMul, ToPrimitive};
use vortex_dtype::{DType, DecimalDType, NativePType, Nullability, match_each_native_ptype};
use vortex_dtype::{DType, DecimalDType, NativePType, Nullability, i256, match_each_native_ptype};
use vortex_error::{VortexExpect, VortexResult, vortex_bail, vortex_err};
use vortex_scalar::{
DecimalScalar, DecimalValue, FromPrimitiveOrF16, PrimitiveScalar, Scalar, ScalarValue, i256,
DecimalScalar, DecimalValue, FromPrimitiveOrF16, PrimitiveScalar, Scalar, ScalarValue,
};

use crate::arrays::{ConstantArray, ConstantVTable};
Expand Down
23 changes: 13 additions & 10 deletions vortex-array/src/arrays/decimal/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,12 @@

use itertools::Itertools;
use vortex_buffer::{BitBufferMut, Buffer, BufferMut, ByteBuffer};
use vortex_dtype::{DType, DecimalDType, IntegerPType, match_each_integer_ptype};
use vortex_dtype::{
BigCast, DType, DecimalDType, DecimalType, IntegerPType, NativeDecimalType,
match_each_integer_ptype,
};
use vortex_error::{VortexExpect, VortexResult, vortex_ensure, vortex_panic};
use vortex_scalar::{BigCast, DecimalValueType, NativeDecimalType, match_each_decimal_value_type};
use vortex_scalar::match_each_decimal_value_type;

use crate::ToCanonical;
use crate::arrays::is_compatible_decimal_value_type;
Expand Down Expand Up @@ -51,7 +54,7 @@ use crate::vtable::ValidityHelper;
/// ## Valid Scalar Types
///
/// The underlying storage uses these native types based on precision:
/// - `DecimalValueType::I8`, `I16`, `I32`, `I64`, `I128`, `I256`
/// - `DecimalType::I8`, `I16`, `I32`, `I64`, `I128`, `I256`
/// - Type selection is automatic based on the required precision
///
/// # Examples
Expand All @@ -75,7 +78,7 @@ use crate::vtable::ValidityHelper;
pub struct DecimalArray {
pub(super) dtype: DType,
pub(super) values: ByteBuffer,
pub(super) values_type: DecimalValueType,
pub(super) values_type: DecimalType,
pub(super) validity: Validity,
pub(super) stats_set: ArrayStats,
}
Expand Down Expand Up @@ -139,7 +142,7 @@ impl DecimalArray {

Self {
values: buffer.into_byte_buffer(),
values_type: T::VALUES_TYPE,
values_type: T::DECIMAL_TYPE,
dtype: DType::Decimal(decimal_dtype, validity.nullability()),
validity,
stats_set: Default::default(),
Expand Down Expand Up @@ -171,10 +174,10 @@ impl DecimalArray {
}

pub fn buffer<T: NativeDecimalType>(&self) -> Buffer<T> {
if self.values_type != T::VALUES_TYPE {
if self.values_type != T::DECIMAL_TYPE {
vortex_panic!(
"Cannot extract Buffer<{:?}> for DecimalArray with values_type {:?}",
T::VALUES_TYPE,
T::DECIMAL_TYPE,
self.values_type,
);
}
Expand All @@ -190,7 +193,7 @@ impl DecimalArray {
}
}

pub fn values_type(&self) -> DecimalValueType {
pub fn values_type(&self) -> DecimalType {
self.values_type
}

Expand Down Expand Up @@ -276,10 +279,10 @@ where
PatchDVT: NativeDecimalType,
ValuesDVT: NativeDecimalType,
{
if !is_compatible_decimal_value_type(ValuesDVT::VALUES_TYPE, decimal_dtype) {
if !is_compatible_decimal_value_type(ValuesDVT::DECIMAL_TYPE, decimal_dtype) {
vortex_panic!(
"patch_typed: {:?} cannot represent every value in {}.",
ValuesDVT::VALUES_TYPE,
ValuesDVT::DECIMAL_TYPE,
decimal_dtype
)
}
Expand Down
8 changes: 4 additions & 4 deletions vortex-array/src/arrays/decimal/compute/between.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_buffer::BitBuffer;
use vortex_dtype::Nullability;
use vortex_dtype::{NativeDecimalType, Nullability};
use vortex_error::{VortexResult, vortex_bail};
use vortex_scalar::{NativeDecimalType, Scalar, match_each_decimal_value_type};
use vortex_scalar::{Scalar, match_each_decimal_value_type};

use crate::arrays::{BoolArray, DecimalArray, DecimalVTable};
use crate::compute::{BetweenKernel, BetweenKernelAdapter, BetweenOptions, StrictComparison};
Expand Down Expand Up @@ -51,7 +51,7 @@ fn between_unpack<T: NativeDecimalType>(
else {
vortex_bail!(
"invalid lower bound Scalar: {lower}, expected {:?}",
T::VALUES_TYPE
T::DECIMAL_TYPE
)
};
let Some(upper_value) = upper
Expand All @@ -61,7 +61,7 @@ fn between_unpack<T: NativeDecimalType>(
else {
vortex_bail!(
"invalid upper bound Scalar: {upper}, expected {:?}",
T::VALUES_TYPE
T::DECIMAL_TYPE
)
};

Expand Down
3 changes: 2 additions & 1 deletion vortex-array/src/arrays/decimal/compute/is_sorted.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use itertools::Itertools;
use vortex_dtype::NativeDecimalType;
use vortex_error::VortexResult;
use vortex_mask::Mask;
use vortex_scalar::{NativeDecimalType, match_each_decimal_value_type};
use vortex_scalar::match_each_decimal_value_type;

use crate::arrays::{DecimalArray, DecimalVTable};
use crate::compute::{IsSortedIteratorExt, IsSortedKernel, IsSortedKernelAdapter};
Expand Down
6 changes: 2 additions & 4 deletions vortex-array/src/arrays/decimal/compute/min_max.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,10 @@
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use itertools::Itertools;
use vortex_dtype::DType;
use vortex_dtype::{DType, NativeDecimalType};
use vortex_error::VortexResult;
use vortex_mask::Mask;
use vortex_scalar::{
DecimalValue, NativeDecimalType, Scalar, ScalarValue, match_each_decimal_value_type,
};
use vortex_scalar::{DecimalValue, Scalar, ScalarValue, match_each_decimal_value_type};

use crate::arrays::{DecimalArray, DecimalVTable};
use crate::compute::{MinMaxKernel, MinMaxKernelAdapter, MinMaxResult};
Expand Down
4 changes: 2 additions & 2 deletions vortex-array/src/arrays/decimal/compute/take.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_buffer::Buffer;
use vortex_dtype::{IntegerPType, match_each_integer_ptype};
use vortex_dtype::{IntegerPType, NativeDecimalType, match_each_integer_ptype};
use vortex_error::VortexResult;
use vortex_scalar::{NativeDecimalType, match_each_decimal_value_type};
use vortex_scalar::match_each_decimal_value_type;

use crate::arrays::{DecimalArray, DecimalVTable};
use crate::compute::{TakeKernel, TakeKernelAdapter};
Expand Down
33 changes: 17 additions & 16 deletions vortex-array/src/arrays/decimal/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,32 +4,34 @@
use itertools::{Itertools, MinMaxResult};
use vortex_dtype::DecimalDType;
use vortex_error::VortexExpect;
use vortex_scalar::{BigCast, DecimalValueType, i256};
use vortex_scalar::{DecimalType, i256};

use crate::arrays::DecimalArray;
use crate::vtable::ValidityHelper;

/// Maps a decimal precision into the smallest type that can represent it.
pub fn smallest_decimal_value_type(decimal_dtype: &DecimalDType) -> DecimalValueType {
pub fn smallest_decimal_value_type(decimal_dtype: &DecimalDType) -> DecimalType {
match decimal_dtype.precision() {
1..=2 => DecimalValueType::I8,
3..=4 => DecimalValueType::I16,
5..=9 => DecimalValueType::I32,
10..=18 => DecimalValueType::I64,
19..=38 => DecimalValueType::I128,
39..=76 => DecimalValueType::I256,
1..=2 => DecimalType::I8,
3..=4 => DecimalType::I16,
5..=9 => DecimalType::I32,
10..=18 => DecimalType::I64,
19..=38 => DecimalType::I128,
39..=76 => DecimalType::I256,
0 => unreachable!("precision must be greater than 0"),
p => unreachable!("precision larger than 76 is invalid found precision {p}"),
}
}

/// True if `value_type` can represent every value of the type `dtype`.
pub fn is_compatible_decimal_value_type(value_type: DecimalValueType, dtype: DecimalDType) -> bool {
pub fn is_compatible_decimal_value_type(value_type: DecimalType, dtype: DecimalDType) -> bool {
value_type >= smallest_decimal_value_type(&dtype)
}

macro_rules! try_downcast {
($array:expr, from: $src:ty, to: $($dst:ty),*) => {{
use vortex_dtype::BigCast;

// Collect the min/max of the values
let minmax = $array.buffer::<$src>().iter().copied().minmax();
match minmax {
Expand Down Expand Up @@ -60,22 +62,21 @@ macro_rules! try_downcast {
pub fn narrowed_decimal(decimal_array: DecimalArray) -> DecimalArray {
match decimal_array.values_type() {
// Cannot narrow any more
DecimalValueType::I8 => decimal_array,
DecimalValueType::I16 => {
DecimalType::I8 => decimal_array,
DecimalType::I16 => {
try_downcast!(decimal_array, from: i16, to: i8)
}
DecimalValueType::I32 => {
DecimalType::I32 => {
try_downcast!(decimal_array, from: i32, to: i8, i16)
}
DecimalValueType::I64 => {
DecimalType::I64 => {
try_downcast!(decimal_array, from: i64, to: i8, i16, i32)
}
DecimalValueType::I128 => {
DecimalType::I128 => {
try_downcast!(decimal_array, from: i128, to: i8, i16, i32, i64)
}
DecimalValueType::I256 => {
DecimalType::I256 => {
try_downcast!(decimal_array, from: i256, to: i8, i16, i32, i64, i128)
}
_ => decimal_array,
}
}
16 changes: 7 additions & 9 deletions vortex-array/src/arrays/decimal/vtable/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@
use std::hash::Hash;

use vortex_dtype::DType;
use vortex_error::vortex_panic;
use vortex_scalar::DecimalValueType;
use vortex_scalar::DecimalType;

use crate::Precision;
use crate::arrays::{DecimalArray, DecimalVTable};
Expand All @@ -16,13 +15,12 @@ use crate::vtable::ArrayVTable;
impl ArrayVTable<DecimalVTable> for DecimalVTable {
fn len(array: &DecimalArray) -> usize {
let divisor = match array.values_type {
DecimalValueType::I8 => 1,
DecimalValueType::I16 => 2,
DecimalValueType::I32 => 4,
DecimalValueType::I64 => 8,
DecimalValueType::I128 => 16,
DecimalValueType::I256 => 32,
ty => vortex_panic!("unknown decimal value type {:?}", ty),
DecimalType::I8 => 1,
DecimalType::I16 => 2,
DecimalType::I32 => 4,
DecimalType::I64 => 8,
DecimalType::I128 => 16,
DecimalType::I256 => 32,
};
array.values.len() / divisor
}
Expand Down
4 changes: 2 additions & 2 deletions vortex-array/src/arrays/decimal/vtable/operations.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
use std::ops::Range;

use vortex_buffer::Buffer;
use vortex_dtype::DecimalDType;
use vortex_scalar::{DecimalValue, NativeDecimalType, Scalar, match_each_decimal_value_type};
use vortex_dtype::{DecimalDType, NativeDecimalType};
use vortex_scalar::{DecimalValue, Scalar, match_each_decimal_value_type};

use crate::arrays::{DecimalArray, DecimalVTable};
use crate::validity::Validity;
Expand Down
8 changes: 4 additions & 4 deletions vortex-array/src/arrays/decimal/vtable/serde.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_buffer::{Alignment, Buffer, ByteBuffer};
use vortex_dtype::DType;
#[cfg(test)]
use vortex_dtype::DecimalDType;
use vortex_dtype::{DType, NativeDecimalType};
use vortex_error::{VortexResult, vortex_bail, vortex_ensure};
use vortex_scalar::{DecimalValueType, NativeDecimalType, match_each_decimal_value_type};
use vortex_scalar::{DecimalType, match_each_decimal_value_type};

use super::{DecimalArray, DecimalEncoding};
use crate::ProstMetadata;
Expand All @@ -18,7 +18,7 @@ use crate::vtable::SerdeVTable;
// The type of the values can be determined by looking at the type info...right?
#[derive(prost::Message)]
pub struct DecimalMetadata {
#[prost(enumeration = "DecimalValueType", tag = "1")]
#[prost(enumeration = "DecimalType", tag = "1")]
pub(super) values_type: i32,
}

Expand Down Expand Up @@ -62,7 +62,7 @@ impl SerdeVTable<DecimalVTable> for DecimalVTable {
vortex_ensure!(
buffer.is_aligned(Alignment::of::<D>()),
"DecimalArray buffer not aligned for values type {:?}",
D::VALUES_TYPE
D::DECIMAL_TYPE
);
let buffer = Buffer::<D>::from_byte_buffer(buffer);
DecimalArray::try_new::<D>(buffer, *decimal_dtype, validity)
Expand Down
Loading
Loading