Add ArrayHash and ArrayEq traits

Cargo.toml

@@ -115,7 +115,7 @@ enum-iterator = "2.0.0"
 erased-serde = "0.4"
 fastlanes = "0.5"
 flatbuffers = "25.2.10"
-fsst-rs = "0.5.2"
+fsst-rs = "0.5.5"
 futures = { version = "0.3.31", default-features = false }
 fuzzy-matcher = "0.3"
 glob = "0.3.2"

encodings/alp/src/alp/array.rs

@@ -2,13 +2,16 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 use std::fmt::Debug;
+use std::hash::Hash;
 
 use vortex_array::patches::Patches;
 use vortex_array::stats::{ArrayStats, StatsSetRef};
 use vortex_array::vtable::{
     ArrayVTable, CanonicalVTable, NotSupported, VTable, ValidityChild, ValidityVTableFromChild,
 };
-use vortex_array::{Array, ArrayRef, Canonical, EncodingId, EncodingRef, vtable};
+use vortex_array::{
+    Array, ArrayEq, ArrayHash, ArrayRef, Canonical, EncodingId, EncodingRef, Precision, vtable,
+};
 use vortex_dtype::{DType, PType};
 use vortex_error::{VortexExpect, VortexResult, vortex_ensure};
 
@@ -261,6 +264,20 @@ impl ArrayVTable<ALPVTable> for ALPVTable {
     fn stats(array: &ALPArray) -> StatsSetRef<'_> {
         array.stats_set.to_ref(array.as_ref())
     }
+
+    fn array_hash<H: std::hash::Hasher>(array: &ALPArray, state: &mut H, precision: Precision) {
+        array.dtype.hash(state);
+        array.encoded.array_hash(state, precision);
+        array.exponents.hash(state);
+        array.patches.array_hash(state, precision);
+    }
+
+    fn array_eq(array: &ALPArray, other: &ALPArray, precision: Precision) -> bool {
+        array.dtype == other.dtype
+            && array.encoded.array_eq(&other.encoded, precision)
+            && array.exponents == other.exponents
+            && array.patches.array_eq(&other.patches, precision)
+    }
 }
 
 impl CanonicalVTable<ALPVTable> for ALPVTable {

encodings/alp/src/alp/mod.rs

@@ -21,7 +21,7 @@ use vortex_scalar::PValue;
 
 const SAMPLE_SIZE: usize = 32;
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct Exponents {
     pub e: u8,
     pub f: u8,

encodings/alp/src/alp_rd/array.rs

@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 use std::fmt::Debug;
+use std::hash::Hash;
 
 use vortex_array::arrays::PrimitiveArray;
 use vortex_array::patches::Patches;
@@ -10,7 +11,10 @@ use vortex_array::validity::Validity;
 use vortex_array::vtable::{
     ArrayVTable, CanonicalVTable, NotSupported, VTable, ValidityChild, ValidityVTableFromChild,
 };
-use vortex_array::{Array, ArrayRef, Canonical, EncodingId, EncodingRef, ToCanonical, vtable};
+use vortex_array::{
+    Array, ArrayEq, ArrayHash, ArrayRef, Canonical, EncodingId, EncodingRef, Precision,
+    ToCanonical, vtable,
+};
 use vortex_buffer::Buffer;
 use vortex_dtype::{DType, PType};
 use vortex_error::{VortexResult, vortex_bail};
@@ -198,6 +202,28 @@ impl ArrayVTable<ALPRDVTable> for ALPRDVTable {
     fn stats(array: &ALPRDArray) -> StatsSetRef<'_> {
         array.stats_set.to_ref(array.as_ref())
     }
+
+    fn array_hash<H: std::hash::Hasher>(array: &ALPRDArray, state: &mut H, precision: Precision) {
+        array.dtype.hash(state);
+        array.left_parts.array_hash(state, precision);
+        array.left_parts_dictionary.array_hash(state, precision);
+        array.right_parts.array_hash(state, precision);
+        array.right_bit_width.hash(state);
+        array.left_parts_patches.array_hash(state, precision);
+    }
+
+    fn array_eq(array: &ALPRDArray, other: &ALPRDArray, precision: Precision) -> bool {
+        array.dtype == other.dtype
+            && array.left_parts.array_eq(&other.left_parts, precision)
+            && array
+                .left_parts_dictionary
+                .array_eq(&other.left_parts_dictionary, precision)
+            && array.right_parts.array_eq(&other.right_parts, precision)
+            && array.right_bit_width == other.right_bit_width
+            && array
+                .left_parts_patches
+                .array_eq(&other.left_parts_patches, precision)
+    }
 }
 
 impl CanonicalVTable<ALPRDVTable> for ALPRDVTable {

encodings/bytebool/src/array.rs

@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 use std::fmt::Debug;
+use std::hash::Hash;
 use std::ops::Range;
 
 use vortex_array::arrays::BoolArray;
@@ -11,7 +12,9 @@ use vortex_array::vtable::{
     ArrayVTable, CanonicalVTable, NotSupported, OperationsVTable, VTable, ValidityHelper,
     ValidityVTableFromValidityHelper,
 };
-use vortex_array::{ArrayRef, Canonical, EncodingId, EncodingRef, IntoArray, vtable};
+use vortex_array::{
+    ArrayEq, ArrayHash, ArrayRef, Canonical, EncodingId, EncodingRef, IntoArray, Precision, vtable,
+};
 use vortex_buffer::{BitBuffer, ByteBuffer};
 use vortex_dtype::DType;
 use vortex_error::vortex_panic;
@@ -109,6 +112,22 @@ impl ArrayVTable<ByteBoolVTable> for ByteBoolVTable {
     fn stats(array: &ByteBoolArray) -> StatsSetRef<'_> {
         array.stats_set.to_ref(array.as_ref())
     }
+
+    fn array_hash<H: std::hash::Hasher>(
+        array: &ByteBoolArray,
+        state: &mut H,
+        precision: Precision,
+    ) {
+        array.dtype.hash(state);
+        array.buffer.array_hash(state, precision);
+        array.validity.array_hash(state, precision);
+    }
+
+    fn array_eq(array: &ByteBoolArray, other: &ByteBoolArray, precision: Precision) -> bool {
+        array.dtype == other.dtype
+            && array.buffer.array_eq(&other.buffer, precision)
+            && array.validity.array_eq(&other.validity, precision)
+    }
 }
 
 impl CanonicalVTable<ByteBoolVTable> for ByteBoolVTable {

encodings/datetime-parts/src/array.rs

@@ -2,12 +2,15 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 use std::fmt::Debug;
+use std::hash::Hash;
 
 use vortex_array::stats::{ArrayStats, StatsSetRef};
 use vortex_array::vtable::{
     ArrayVTable, NotSupported, VTable, ValidityChild, ValidityVTableFromChild,
 };
-use vortex_array::{Array, ArrayRef, EncodingId, EncodingRef, vtable};
+use vortex_array::{
+    Array, ArrayEq, ArrayHash, ArrayRef, EncodingId, EncodingRef, Precision, vtable,
+};
 use vortex_dtype::DType;
 use vortex_error::{VortexResult, vortex_bail};
 
@@ -128,6 +131,28 @@ impl ArrayVTable<DateTimePartsVTable> for DateTimePartsVTable {
     fn stats(array: &DateTimePartsArray) -> StatsSetRef<'_> {
         array.stats_set.to_ref(array.as_ref())
     }
+
+    fn array_hash<H: std::hash::Hasher>(
+        array: &DateTimePartsArray,
+        state: &mut H,
+        precision: Precision,
+    ) {
+        array.dtype.hash(state);
+        array.days.array_hash(state, precision);
+        array.seconds.array_hash(state, precision);
+        array.subseconds.array_hash(state, precision);
+    }
+
+    fn array_eq(
+        array: &DateTimePartsArray,
+        other: &DateTimePartsArray,
+        precision: Precision,
+    ) -> bool {
+        array.dtype == other.dtype
+            && array.days.array_eq(&other.days, precision)
+            && array.seconds.array_eq(&other.seconds, precision)
+            && array.subseconds.array_eq(&other.subseconds, precision)
+    }
 }
 
 impl ValidityChild<DateTimePartsVTable> for DateTimePartsVTable {

encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs

@@ -4,6 +4,7 @@
 mod compute;
 mod serde;
 
+use std::hash::Hash;
 use std::ops::Range;
 
 use vortex_array::arrays::DecimalArray;
@@ -13,7 +14,8 @@ use vortex_array::vtable::{
     ValidityHelper, ValidityVTableFromChild,
 };
 use vortex_array::{
-    Array, ArrayRef, Canonical, EncodingId, EncodingRef, IntoArray, ToCanonical, vtable,
+    Array, ArrayEq, ArrayHash, ArrayRef, Canonical, EncodingId, EncodingRef, IntoArray, Precision,
+    ToCanonical, vtable,
 };
 use vortex_dtype::{DType, DecimalDType, match_each_signed_integer_ptype};
 use vortex_error::{VortexExpect, VortexResult, vortex_bail};
@@ -111,6 +113,23 @@ impl ArrayVTable<DecimalBytePartsVTable> for DecimalBytePartsVTable {
     fn stats(array: &DecimalBytePartsArray) -> StatsSetRef<'_> {
         array.stats_set.to_ref(array.as_ref())
     }
+
+    fn array_hash<H: std::hash::Hasher>(
+        array: &DecimalBytePartsArray,
+        state: &mut H,
+        precision: Precision,
+    ) {
+        array.dtype.hash(state);
+        array.msp.array_hash(state, precision);
+    }
+
+    fn array_eq(
+        array: &DecimalBytePartsArray,
+        other: &DecimalBytePartsArray,
+        precision: Precision,
+    ) -> bool {
+        array.dtype == other.dtype && array.msp.array_eq(&other.msp, precision)
+    }
 }
 
 impl CanonicalVTable<DecimalBytePartsVTable> for DecimalBytePartsVTable {

encodings/dict/src/array.rs

@@ -2,10 +2,13 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 use std::fmt::Debug;
+use std::hash::Hash;
 
 use vortex_array::stats::{ArrayStats, StatsSetRef};
 use vortex_array::vtable::{ArrayVTable, NotSupported, VTable, ValidityVTable};
-use vortex_array::{Array, ArrayRef, EncodingId, EncodingRef, ToCanonical, vtable};
+use vortex_array::{
+    Array, ArrayEq, ArrayHash, ArrayRef, EncodingId, EncodingRef, Precision, ToCanonical, vtable,
+};
 use vortex_buffer::BitBuffer;
 use vortex_dtype::{DType, match_each_integer_ptype};
 use vortex_error::{VortexExpect as _, VortexResult, vortex_bail};
@@ -116,6 +119,18 @@ impl ArrayVTable<DictVTable> for DictVTable {
     fn stats(array: &DictArray) -> StatsSetRef<'_> {
         array.stats_set.to_ref(array.as_ref())
     }
+
+    fn array_hash<H: std::hash::Hasher>(array: &DictArray, state: &mut H, precision: Precision) {
+        array.dtype.hash(state);
+        array.codes.array_hash(state, precision);
+        array.values.array_hash(state, precision);
+    }
+
+    fn array_eq(array: &DictArray, other: &DictArray, precision: Precision) -> bool {
+        array.dtype == other.dtype
+            && array.codes.array_eq(&other.codes, precision)
+            && array.values.array_eq(&other.values, precision)
+    }
 }
 
 impl ValidityVTable<DictVTable> for DictVTable {

encodings/fastlanes/src/bitpacking/mod.rs

@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 use std::fmt::Debug;
+use std::hash::Hash;
 
 pub use compress::*;
 use fastlanes::BitPacking;
@@ -14,7 +15,9 @@ use vortex_array::vtable::{
     ArrayVTable, CanonicalVTable, NotSupported, VTable, ValidityHelper,
     ValidityVTableFromValidityHelper,
 };
-use vortex_array::{Array, Canonical, EncodingId, EncodingRef, vtable};
+use vortex_array::{
+    Array, ArrayEq, ArrayHash, Canonical, EncodingId, EncodingRef, Precision, vtable,
+};
 use vortex_buffer::ByteBuffer;
 use vortex_dtype::{DType, NativePType, PType, match_each_integer_ptype};
 use vortex_error::{VortexExpect, VortexResult, vortex_bail, vortex_ensure};
@@ -330,6 +333,30 @@ impl ArrayVTable<BitPackedVTable> for BitPackedVTable {
     fn stats(array: &BitPackedArray) -> StatsSetRef<'_> {
         array.stats_set.to_ref(array.as_ref())
     }
+
+    fn array_hash<H: std::hash::Hasher>(
+        array: &BitPackedArray,
+        state: &mut H,
+        precision: Precision,
+    ) {
+        array.offset.hash(state);
+        array.len.hash(state);
+        array.dtype.hash(state);
+        array.bit_width.hash(state);
+        array.packed.array_hash(state, precision);
+        array.patches.array_hash(state, precision);
+        array.validity.array_hash(state, precision);
+    }
+
+    fn array_eq(array: &BitPackedArray, other: &BitPackedArray, precision: Precision) -> bool {
+        array.offset == other.offset
+            && array.len == other.len
+            && array.dtype == other.dtype
+            && array.bit_width == other.bit_width
+            && array.packed.array_eq(&other.packed, precision)
+            && array.patches.array_eq(&other.patches, precision)
+            && array.validity.array_eq(&other.validity, precision)
+    }
 }
 
 impl CanonicalVTable<BitPackedVTable> for BitPackedVTable {

encodings/fastlanes/src/delta/mod.rs

@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 use std::fmt::Debug;
+use std::hash::Hash;
 
 pub use compress::*;
 use fastlanes::FastLanes;
@@ -12,7 +13,10 @@ use vortex_array::vtable::{
     ArrayVTable, CanonicalVTable, NotSupported, VTable, ValidityChildSliceHelper,
     ValidityVTableFromChildSliceHelper,
 };
-use vortex_array::{Array, ArrayRef, Canonical, EncodingId, EncodingRef, IntoArray, vtable};
+use vortex_array::{
+    Array, ArrayEq, ArrayHash, ArrayRef, Canonical, EncodingId, EncodingRef, IntoArray, Precision,
+    vtable,
+};
 use vortex_buffer::Buffer;
 use vortex_dtype::{DType, NativePType, PType, match_each_unsigned_integer_ptype};
 use vortex_error::{VortexExpect as _, VortexResult, vortex_bail};
@@ -237,6 +241,22 @@ impl ArrayVTable<DeltaVTable> for DeltaVTable {
     fn stats(array: &DeltaArray) -> StatsSetRef<'_> {
         array.stats_set.to_ref(array.as_ref())
     }
+
+    fn array_hash<H: std::hash::Hasher>(array: &DeltaArray, state: &mut H, precision: Precision) {
+        array.offset.hash(state);
+        array.len.hash(state);
+        array.dtype.hash(state);
+        array.bases.array_hash(state, precision);
+        array.deltas.array_hash(state, precision);
+    }
+
+    fn array_eq(array: &DeltaArray, other: &DeltaArray, precision: Precision) -> bool {
+        array.offset == other.offset
+            && array.len == other.len
+            && array.dtype == other.dtype
+            && array.bases.array_eq(&other.bases, precision)
+            && array.deltas.array_eq(&other.deltas, precision)
+    }
 }
 
 impl CanonicalVTable<DeltaVTable> for DeltaVTable {