From 28c556a10baa41f7c59a7181971f80313561a392 Mon Sep 17 00:00:00 2001 From: Simon Lin Date: Mon, 30 Sep 2024 20:02:06 +1000 Subject: [PATCH] refactor: Fix new-streaming `test_lazy_parquet::test_row_index` --- crates/polars-python/Cargo.toml | 2 +- .../polars-stream/src/nodes/parquet_source/row_group_decode.rs | 2 -- py-polars/polars/lazyframe/frame.py | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/polars-python/Cargo.toml b/crates/polars-python/Cargo.toml index 9ed35648c89f4..db8d2bb68042e 100644 --- a/crates/polars-python/Cargo.toml +++ b/crates/polars-python/Cargo.toml @@ -247,7 +247,7 @@ all = [ "binary_encoding", "ffi_plugin", "polars_cloud", - # "new_streaming", + "new_streaming", ] # we cannot conditionally activate simd diff --git a/crates/polars-stream/src/nodes/parquet_source/row_group_decode.rs b/crates/polars-stream/src/nodes/parquet_source/row_group_decode.rs index a0f48b70f919e..dc8fe611eafde 100644 --- a/crates/polars-stream/src/nodes/parquet_source/row_group_decode.rs +++ b/crates/polars-stream/src/nodes/parquet_source/row_group_decode.rs @@ -639,8 +639,6 @@ fn decode_column_prefiltered( deserialize_filter, )?; - assert_eq!(array.len(), expected_num_rows); - let column = Series::try_from((arrow_field, array))?.into_column(); let column = if !prefilter { diff --git a/py-polars/polars/lazyframe/frame.py b/py-polars/polars/lazyframe/frame.py index 9e65f918d3857..1426409d0ed68 100644 --- a/py-polars/polars/lazyframe/frame.py +++ b/py-polars/polars/lazyframe/frame.py @@ -1983,6 +1983,7 @@ def collect( └─────┴─────┴─────┘ """ new_streaming = _kwargs.get("new_streaming", False) + new_streaming = _kwargs.get("new_streaming", True) if no_optimization or _eager: predicate_pushdown = False