Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
2 changes: 1 addition & 1 deletion bench-vortex/rustfmt.toml
Original file line number Diff line number Diff line change
@@ -1 +1 @@
imports_granularity = "Module"
imports_granularity = "Module"
12 changes: 9 additions & 3 deletions encodings/alp/benches/alp_compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,19 @@
#![allow(clippy::unwrap_used)]

use divan::Bencher;
use rand::Rng;
use rand::SeedableRng as _;
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng as _};
use vortex_alp::{ALPFloat, ALPRDFloat, RDEncoder, alp_encode, decompress};
use vortex_alp::ALPFloat;
use vortex_alp::ALPRDFloat;
use vortex_alp::RDEncoder;
use vortex_alp::alp_encode;
use vortex_alp::decompress;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::compute::warm_up_vtables;
use vortex_array::validity::Validity;
use vortex_buffer::{Buffer, buffer};
use vortex_buffer::Buffer;
use vortex_buffer::buffer;
use vortex_dtype::NativePType;

fn main() {
Expand Down
1 change: 0 additions & 1 deletion encodings/alp/rustfmt.toml

This file was deleted.

50 changes: 37 additions & 13 deletions encodings/alp/src/alp/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,47 @@
use std::fmt::Debug;
use std::hash::Hash;

use vortex_array::patches::{Patches, PatchesMetadata};
use vortex_array::Array;
use vortex_array::ArrayBufferVisitor;
use vortex_array::ArrayChildVisitor;
use vortex_array::ArrayEq;
use vortex_array::ArrayHash;
use vortex_array::ArrayRef;
use vortex_array::Canonical;
use vortex_array::DeserializeMetadata;
use vortex_array::Precision;
use vortex_array::ProstMetadata;
use vortex_array::SerializeMetadata;
use vortex_array::patches::Patches;
use vortex_array::patches::PatchesMetadata;
use vortex_array::serde::ArrayChildren;
use vortex_array::stats::{ArrayStats, StatsSetRef};
use vortex_array::vtable::{
ArrayId, ArrayVTable, ArrayVTableExt, BaseArrayVTable, CanonicalVTable, EncodeVTable,
NotSupported, VTable, ValidityChild, ValidityVTableFromChild, VisitorVTable,
};
use vortex_array::{
Array, ArrayBufferVisitor, ArrayChildVisitor, ArrayEq, ArrayHash, ArrayRef, Canonical,
DeserializeMetadata, Precision, ProstMetadata, SerializeMetadata, vtable,
};
use vortex_array::stats::ArrayStats;
use vortex_array::stats::StatsSetRef;
use vortex_array::vtable;
use vortex_array::vtable::ArrayId;
use vortex_array::vtable::ArrayVTable;
use vortex_array::vtable::ArrayVTableExt;
use vortex_array::vtable::BaseArrayVTable;
use vortex_array::vtable::CanonicalVTable;
use vortex_array::vtable::EncodeVTable;
use vortex_array::vtable::NotSupported;
use vortex_array::vtable::VTable;
use vortex_array::vtable::ValidityChild;
use vortex_array::vtable::ValidityVTableFromChild;
use vortex_array::vtable::VisitorVTable;
use vortex_buffer::ByteBuffer;
use vortex_dtype::{DType, PType};
use vortex_error::{VortexError, VortexExpect, VortexResult, vortex_bail, vortex_ensure};
use vortex_dtype::DType;
use vortex_dtype::PType;
use vortex_error::VortexError;
use vortex_error::VortexExpect;
use vortex_error::VortexResult;
use vortex_error::vortex_bail;
use vortex_error::vortex_ensure;

use crate::ALPFloat;
use crate::alp::{Exponents, alp_encode, decompress};
use crate::alp::Exponents;
use crate::alp::alp_encode;
use crate::alp::decompress;

vtable!(ALP);

Expand Down
25 changes: 17 additions & 8 deletions encodings/alp/src/alp/compress.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,25 @@ use std::mem::transmute;

use itertools::Itertools;
use num_traits::AsPrimitive;
use vortex_array::arrays::{PrimitiveArray, patch_chunk};
use vortex_array::ArrayRef;
use vortex_array::IntoArray;
use vortex_array::ToCanonical;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::arrays::patch_chunk;
use vortex_array::patches::Patches;
use vortex_array::validity::Validity;
use vortex_array::vtable::ValidityHelper;
use vortex_array::{ArrayRef, IntoArray, ToCanonical};
use vortex_buffer::{Buffer, BufferMut};
use vortex_dtype::{PType, match_each_unsigned_integer_ptype};
use vortex_error::{VortexResult, vortex_bail};
use vortex_buffer::Buffer;
use vortex_buffer::BufferMut;
use vortex_dtype::PType;
use vortex_dtype::match_each_unsigned_integer_ptype;
use vortex_error::VortexResult;
use vortex_error::vortex_bail;
use vortex_mask::Mask;

use crate::Exponents;
use crate::alp::{ALPArray, ALPFloat};
use crate::alp::ALPArray;
use crate::alp::ALPFloat;

#[macro_export]
macro_rules! match_each_alp_float_ptype {
Expand Down Expand Up @@ -246,10 +253,12 @@ fn decompress_unchunked(array: ALPArray) -> PrimitiveArray {
mod tests {
use core::f64;

use f64::consts::{E, PI};
use f64::consts::E;
use f64::consts::PI;
use vortex_array::assert_arrays_eq;
use vortex_array::validity::Validity;
use vortex_buffer::{Buffer, buffer};
use vortex_buffer::Buffer;
use vortex_buffer::buffer;
use vortex_dtype::NativePType;

use super::*;
Expand Down
27 changes: 19 additions & 8 deletions encodings/alp/src/alp/compute/between.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,25 @@

use std::fmt::Debug;

use vortex_array::Array;
use vortex_array::ArrayRef;
use vortex_array::arrays::ConstantArray;
use vortex_array::compute::{
BetweenKernel, BetweenKernelAdapter, BetweenOptions, StrictComparison, between,
};
use vortex_array::{Array, ArrayRef, register_kernel};
use vortex_dtype::{NativeDType, NativePType, Nullability};
use vortex_array::compute::BetweenKernel;
use vortex_array::compute::BetweenKernelAdapter;
use vortex_array::compute::BetweenOptions;
use vortex_array::compute::StrictComparison;
use vortex_array::compute::between;
use vortex_array::register_kernel;
use vortex_dtype::NativeDType;
use vortex_dtype::NativePType;
use vortex_dtype::Nullability;
use vortex_error::VortexResult;
use vortex_scalar::Scalar;

use crate::{ALPArray, ALPFloat, ALPVTable, match_each_alp_float_ptype};
use crate::ALPArray;
use crate::ALPFloat;
use crate::ALPVTable;
use crate::match_each_alp_float_ptype;

impl BetweenKernel for ALPVTable {
fn between(
Expand Down Expand Up @@ -94,11 +103,13 @@ mod tests {
use itertools::Itertools;
use vortex_array::ToCanonical;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::compute::{BetweenOptions, StrictComparison};
use vortex_array::compute::BetweenOptions;
use vortex_array::compute::StrictComparison;
use vortex_dtype::Nullability;

use crate::ALPArray;
use crate::alp::compute::between::between_impl;
use crate::{ALPArray, alp_encode};
use crate::alp_encode;

fn between_test(arr: &ALPArray, lower: f32, upper: f32, options: &BetweenOptions) -> bool {
let res = between_impl(arr, lower, upper, Nullability::Nullable, options)
Expand Down
18 changes: 13 additions & 5 deletions encodings/alp/src/alp/compute/cast.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_array::compute::{CastKernel, CastKernelAdapter, cast};
use vortex_array::{ArrayRef, IntoArray, register_kernel};
use vortex_array::ArrayRef;
use vortex_array::IntoArray;
use vortex_array::compute::CastKernel;
use vortex_array::compute::CastKernelAdapter;
use vortex_array::compute::cast;
use vortex_array::register_kernel;
use vortex_dtype::DType;
use vortex_error::VortexResult;

use crate::alp::{ALPArray, ALPVTable};
use crate::alp::ALPArray;
use crate::alp::ALPVTable;

impl CastKernel for ALPVTable {
fn cast(&self, array: &ALPArray, dtype: &DType) -> VortexResult<Option<ArrayRef>> {
Expand Down Expand Up @@ -45,13 +50,16 @@ register_kernel!(CastKernelAdapter(ALPVTable).lift());
#[cfg(test)]
mod tests {
use rstest::rstest;
use vortex_array::IntoArray;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::assert_arrays_eq;
use vortex_array::compute::cast;
use vortex_array::compute::conformance::cast::test_cast_conformance;
use vortex_array::vtable::ArrayVTableExt;
use vortex_array::{IntoArray, assert_arrays_eq};
use vortex_buffer::buffer;
use vortex_dtype::{DType, Nullability, PType};
use vortex_dtype::DType;
use vortex_dtype::Nullability;
use vortex_dtype::PType;

use crate::ALPVTable;

Expand Down
31 changes: 23 additions & 8 deletions encodings/alp/src/alp/compute/compare.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,25 @@

use std::fmt::Debug;

use vortex_array::Array;
use vortex_array::ArrayRef;
use vortex_array::IntoArray;
use vortex_array::arrays::ConstantArray;
use vortex_array::compute::{CompareKernel, CompareKernelAdapter, Operator, compare};
use vortex_array::{Array, ArrayRef, IntoArray, register_kernel};
use vortex_array::compute::CompareKernel;
use vortex_array::compute::CompareKernelAdapter;
use vortex_array::compute::Operator;
use vortex_array::compute::compare;
use vortex_array::register_kernel;
use vortex_dtype::NativePType;
use vortex_error::{VortexResult, vortex_bail};
use vortex_scalar::{PrimitiveScalar, Scalar};
use vortex_error::VortexResult;
use vortex_error::vortex_bail;
use vortex_scalar::PrimitiveScalar;
use vortex_scalar::Scalar;

use crate::{ALPArray, ALPFloat, ALPVTable, match_each_alp_float_ptype};
use crate::ALPArray;
use crate::ALPFloat;
use crate::ALPVTable;
use crate::match_each_alp_float_ptype;

// TODO(joe): add fuzzing.

Expand Down Expand Up @@ -129,9 +140,13 @@ where
mod tests {
use rstest::rstest;
use vortex_array::ToCanonical;
use vortex_array::arrays::{ConstantArray, PrimitiveArray};
use vortex_array::compute::{Operator, compare};
use vortex_dtype::{DType, Nullability, PType};
use vortex_array::arrays::ConstantArray;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::compute::Operator;
use vortex_array::compute::compare;
use vortex_dtype::DType;
use vortex_dtype::Nullability;
use vortex_dtype::PType;
use vortex_scalar::Scalar;

use super::*;
Expand Down
13 changes: 9 additions & 4 deletions encodings/alp/src/alp/compute/filter.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_array::compute::{FilterKernel, FilterKernelAdapter, filter};
use vortex_array::{ArrayRef, register_kernel};
use vortex_array::ArrayRef;
use vortex_array::compute::FilterKernel;
use vortex_array::compute::FilterKernelAdapter;
use vortex_array::compute::filter;
use vortex_array::register_kernel;
use vortex_error::VortexResult;
use vortex_mask::Mask;

use crate::{ALPArray, ALPVTable};
use crate::ALPArray;
use crate::ALPVTable;

impl FilterKernel for ALPVTable {
fn filter(&self, array: &ALPArray, mask: &Mask) -> VortexResult<ArrayRef> {
Expand Down Expand Up @@ -34,10 +38,11 @@ register_kernel!(FilterKernelAdapter(ALPVTable).lift());
#[cfg(test)]
mod test {
use rstest::rstest;
use vortex_array::ArrayRef;
use vortex_array::IntoArray;
use vortex_array::arrays::PrimitiveArray;
use vortex_array::compute::conformance::filter::test_filter_conformance;
use vortex_array::vtable::ArrayVTableExt;
use vortex_array::{ArrayRef, IntoArray};
use vortex_buffer::buffer;

use crate::ALPVTable;
Expand Down
10 changes: 7 additions & 3 deletions encodings/alp/src/alp/compute/mask.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_array::compute::{MaskKernel, MaskKernelAdapter, mask};
use vortex_array::{ArrayRef, register_kernel};
use vortex_array::ArrayRef;
use vortex_array::compute::MaskKernel;
use vortex_array::compute::MaskKernelAdapter;
use vortex_array::compute::mask;
use vortex_array::register_kernel;
use vortex_error::VortexResult;
use vortex_mask::Mask;

use crate::{ALPArray, ALPVTable};
use crate::ALPArray;
use crate::ALPVTable;

impl MaskKernel for ALPVTable {
fn mask(&self, array: &ALPArray, filter_mask: &Mask) -> VortexResult<ArrayRef> {
Expand Down
3 changes: 2 additions & 1 deletion encodings/alp/src/alp/compute/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ mod tests {
use vortex_array::compute::conformance::binary_numeric::test_binary_numeric_array;
use vortex_array::compute::conformance::consistency::test_array_consistency;

use crate::{ALPArray, alp_encode};
use crate::ALPArray;
use crate::alp_encode;

#[rstest]
// Basic float arrays
Expand Down
7 changes: 5 additions & 2 deletions encodings/alp/src/alp/compute/nan_count.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_array::compute::{NaNCountKernel, NaNCountKernelAdapter, nan_count};
use vortex_array::compute::NaNCountKernel;
use vortex_array::compute::NaNCountKernelAdapter;
use vortex_array::compute::nan_count;
use vortex_array::register_kernel;
use vortex_error::VortexResult;

use crate::{ALPArray, ALPVTable};
use crate::ALPArray;
use crate::ALPVTable;

impl NaNCountKernel for ALPVTable {
fn nan_count(&self, array: &ALPArray) -> VortexResult<usize> {
Expand Down
12 changes: 9 additions & 3 deletions encodings/alp/src/alp/compute/take.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,17 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

use vortex_array::compute::{TakeKernel, TakeKernelAdapter, take};
use vortex_array::{Array, ArrayRef, IntoArray, register_kernel};
use vortex_array::Array;
use vortex_array::ArrayRef;
use vortex_array::IntoArray;
use vortex_array::compute::TakeKernel;
use vortex_array::compute::TakeKernelAdapter;
use vortex_array::compute::take;
use vortex_array::register_kernel;
use vortex_error::VortexResult;

use crate::{ALPArray, ALPVTable};
use crate::ALPArray;
use crate::ALPVTable;

impl TakeKernel for ALPVTable {
fn take(&self, array: &ALPArray, indices: &dyn Array) -> VortexResult<ArrayRef> {
Expand Down
Loading
Loading