Upgrade arrow2, arrow2_convert and polars (#665)

* Upgrade arrow2, arrow2_convert and polars including unmerged/unreleased PRs

Cargo.toml

@@ -13,13 +13,14 @@ repository = "https://github.com/rerun-io/rerun"
 
 [workspace.dependencies]
 anyhow = "1.0"
-arrow2 = "0.14"
+arrow2 = "0.15"
 arrow2_convert = "0.3"
 comfy-table = { version = "6.1", default-features = false }
 ecolor = { version = "0.20" }
 lazy_static = "1.4"
-polars-core = "0.25"
-polars-lazy = "0.25"
+polars-core = "0.26"
+polars-lazy = "0.26"
+polars-ops = "0.26"
 puffin = "0.14"
 thiserror = "1.0"
 
@@ -62,6 +63,9 @@ wgpu = { git = "https://github.com/gfx-rs/wgpu.git", ref = "a377ae2b7fe6c1c94127
 wgpu-core = { git = "https://github.com/gfx-rs/wgpu.git", ref = "a377ae2b7fe6c1c9412751166f0917e617164e49" }
 #wgpu = { path = "../wgpu/wgpu" }
 
-# Upstream PR https://github.com/jorgecarleitao/arrow2/pull/1334
-arrow2 = { git = "https://github.com/rerun-io/arrow2", rev = "ed459eb9e3b49877e3e827753621d75426a8496d" }
+# Upstream PR https://github.com/jorgecarleitao/arrow2/pull/1334 merged, but not released
+arrow2 = { git = "https://github.com/rerun-io/arrow2", rev = "26d1b1139ac3db22c4973772ddb0d30c16027ff6" }
+# Upstream PRs https://github.com/DataEngineeringLabs/arrow2-convert/pull/88 and https://github.com/DataEngineeringLabs/arrow2-convert/pull/89
+arrow2_convert = { git = "https://github.com/rerun-io/arrow2-convert", rev = "5b98b537d8095fac807bb6f018536125668f7e90" }
 #arrow2 = { path = "../arrow2" }
+#arrow2_convert = { path = "../arrow2-convert/arrow2_convert" }

crates/re_arrow_store/Cargo.toml

@@ -12,7 +12,7 @@ default = []
 ## Enables `parking_lot`'s deadlock detection background thread.
 deadlock_detection = ["parking_lot/deadlock_detection"]
 ## Integration with `polars`, to efficiently use the datastore with dataframes.
-polars = ["dep:polars-core"]
+polars = ["dep:polars-core", "dep:polars-ops"]
 
 
 [dependencies]
@@ -45,16 +45,25 @@ polars-core = { workspace = true, optional = true, features = [
   "dtype-struct",
   "sort_multiple",
 ] }
+polars-ops = { workspace = true, optional = true, features = [
+  "dtype-date",
+  "dtype-datetime",
+  "dtype-time",
+  "dtype-struct",
+] }
 
 
 [dev-dependencies]
 criterion = "0.4"
 mimalloc = "0.1"
 polars-core = { workspace = true, features = [
+  "diagonal_concat",
   "dtype-date",
-  "dtype-time",
+  "dtype-datetime",
   "dtype-struct",
+  "dtype-time",
   "fmt",
+  "sort_multiple",
 ] }
 tracing-subscriber = "0.3"
 

crates/re_arrow_store/src/arrow_util.rs

@@ -57,13 +57,18 @@ impl ArrayExt for dyn Array {
         }
     }
 
+    /// Return the length of the first child.
+    ///
+    /// ## Panics
+    ///
+    /// Panics if `Self` is not a `ListArray<i32>`, or if the array is empty (no children).
     fn get_child_length(&self, child_nr: usize) -> usize {
-        let offsets = self
-            .as_any()
+        self.as_any()
             .downcast_ref::<ListArray<i32>>()
             .unwrap()
-            .offsets();
-
-        (offsets[child_nr + 1] - offsets[child_nr]) as usize
+            .offsets()
+            .lengths()
+            .nth(child_nr)
+            .unwrap()
     }
 }

crates/re_arrow_store/src/polars_util.rs

@@ -1,6 +1,7 @@
 use arrow2::array::Array;
 use itertools::Itertools;
 use polars_core::{prelude::*, series::Series};
+use polars_ops::prelude::*;
 use re_log_types::{ComponentName, ObjPath as EntityPath, TimeInt};
 
 use crate::{DataStore, LatestAtQuery, RangeQuery};

crates/re_arrow_store/src/store.rs

@@ -1079,9 +1079,9 @@ impl ComponentBucket {
 fn test_arrow_estimated_size_bytes() {
     use arrow2::{
         array::{Float64Array, ListArray, StructArray, UInt64Array, Utf8Array},
-        buffer::Buffer,
         compute::aggregate::estimated_bytes_size,
         datatypes::{DataType, Field},
+        offset::Offsets,
     };
 
     // simple primitive array
@@ -1119,16 +1119,11 @@ fn test_arrow_estimated_size_bytes() {
             let array_flattened =
                 UInt64Array::from_vec(data.clone().into_iter().flatten().collect()).boxed();
 
-            let mut i = 0i32;
-            let indices = std::iter::from_fn(move || {
-                let ret = i;
-                i += 50;
-                Some(ret)
-            });
-
-            ListArray::<i32>::from_data(
+            ListArray::<i32>::new(
                 ListArray::<i32>::default_datatype(DataType::UInt64),
-                Buffer::from_iter(indices.take(50)),
+                Offsets::try_from_lengths(std::iter::repeat(50).take(50))
+                    .unwrap()
+                    .into(),
                 array_flattened,
                 None,
             )
@@ -1206,16 +1201,11 @@ fn test_arrow_estimated_size_bytes() {
                 StructArray::new(DataType::Struct(fields), vec![x, y], None)
             };
 
-            let mut i = 0i32;
-            let indices = std::iter::from_fn(move || {
-                let ret = i;
-                i += 50;
-                Some(ret)
-            });
-
-            ListArray::<i32>::from_data(
+            ListArray::<i32>::new(
                 ListArray::<i32>::default_datatype(array.data_type().clone()),
-                Buffer::from_iter(indices.take(50)),
+                Offsets::try_from_lengths(std::iter::repeat(50).take(50))
+                    .unwrap()
+                    .into(),
                 array.boxed(),
                 None,
             )