Skip to content

Commit 005b101

Browse files
committed
use page row limit fix ut
Signed-off-by: yangjiang <yangjiang@ebay.com>
1 parent 1c97dc4 commit 005b101

File tree

2 files changed

+3
-6
lines changed

2 files changed

+3
-6
lines changed

datafusion/core/src/datasource/file_format/parquet.rs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -538,9 +538,7 @@ pub(crate) mod test_util {
538538
// All batches write in to one file, each batch must have same schema.
539539
let mut output = NamedTempFile::new().expect("creating temp file");
540540
let mut builder = WriterProperties::builder();
541-
// todo https://github.com/apache/arrow-rs/issues/2941 release change to row limit.
542-
builder = builder.set_data_pagesize_limit(1);
543-
builder = builder.set_write_batch_size(1);
541+
builder = builder.set_data_page_row_count_limit(2);
544542
let proper = builder.build();
545543
let mut writer =
546544
ArrowWriter::try_new(&mut output, batches[0].schema(), Some(proper))

datafusion/core/src/physical_plan/file_format/parquet.rs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1726,11 +1726,10 @@ mod tests {
17261726
// todo fix this https://github.com/apache/arrow-rs/issues/2941 release change to row limit.
17271727
// assert the batches and some metrics
17281728
let expected = vec![
1729-
"+-----+", "| int |", "+-----+", "| |", "| 1 |", "| 2 |", "| 3 |",
1730-
"| 4 |", "| 5 |", "+-----+",
1729+
"+-----+", "| int |", "+-----+", "| 3 |", "| 4 |", "| 5 |", "+-----+",
17311730
];
17321731
assert_batches_sorted_eq!(expected, &rt.batches.unwrap());
1733-
assert_eq!(get_value(&metrics, "page_index_rows_filtered"), 0);
1732+
assert_eq!(get_value(&metrics, "page_index_rows_filtered"), 3);
17341733
assert!(
17351734
get_value(&metrics, "page_index_eval_time") > 0,
17361735
"no eval time in metrics: {:#?}",

0 commit comments

Comments
 (0)