Skip to content

Commit debfd19

Browse files
connecconnec
committed
chore: replace usage of deprecated CsvExec::new with CsvExec::builder 
1 parent 0c51fdb commit debfd19

File tree

7 files changed

+258
-224
lines changed

7 files changed

+258
-224
lines changed

‎datafusion/core/src/datasource/file_format/csv.rs‎

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -344,22 +344,25 @@ impl FileFormat for CsvFormat {
344344
conf: FileScanConfig,
345345
_filters: Option<&Arc<dyn PhysicalExpr>>,
346346
) -> Result<Arc<dyn ExecutionPlan>> {
347-
let exec = CsvExec::new(
348-
conf,
349-
// If format options does not specify whether there is a header,
350-
// we consult configuration options.
351-
self.options
352-
.has_header
353-
.unwrap_or(state.config_options().catalog.has_header),
354-
self.options.delimiter,
355-
self.options.quote,
356-
self.options.escape,
357-
self.options.comment,
358-
self.options
359-
.newlines_in_values
360-
.unwrap_or(state.config_options().catalog.newlines_in_values),
361-
self.options.compression.into(),
362-
);
347+
// Consult configuration options for default values
348+
let has_header = self
349+
.options
350+
.has_header
351+
.unwrap_or(state.config_options().catalog.has_header);
352+
let newlines_in_values = self
353+
.options
354+
.newlines_in_values
355+
.unwrap_or(state.config_options().catalog.newlines_in_values);
356+
357+
let exec = CsvExec::builder(conf)
358+
.with_has_header(has_header)
359+
.with_delimeter(self.options.delimiter)
360+
.with_quote(self.options.quote)
361+
.with_escape(self.options.escape)
362+
.with_comment(self.options.comment)
363+
.with_newlines_in_values(newlines_in_values)
364+
.with_file_compression_type(self.options.compression.into())
365+
.build();
363366
Ok(Arc::new(exec))
364367
}
365368

‎datafusion/core/src/datasource/physical_plan/csv.rs‎

Lines changed: 54 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -747,16 +747,15 @@ mod tests {
747747
let mut config = partitioned_csv_config(file_schema, file_groups);
748748
config.projection = Some(vec![0, 2, 4]);
749749

750-
let csv = CsvExec::new(
751-
config,
752-
true,
753-
b',',
754-
b'"',
755-
None,
756-
None,
757-
false,
758-
file_compression_type.to_owned(),
759-
);
750+
let csv = CsvExec::builder(config)
751+
.with_has_header(true)
752+
.with_delimeter(b',')
753+
.with_quote(b'"')
754+
.with_escape(None)
755+
.with_comment(None)
756+
.with_newlines_in_values(false)
757+
.with_file_compression_type(file_compression_type)
758+
.build();
760759
assert_eq!(13, csv.base_config.file_schema.fields().len());
761760
assert_eq!(3, csv.schema().fields().len());
762761

@@ -817,16 +816,15 @@ mod tests {
817816
let mut config = partitioned_csv_config(file_schema, file_groups);
818817
config.projection = Some(vec![4, 0, 2]);
819818

820-
let csv = CsvExec::new(
821-
config,
822-
true,
823-
b',',
824-
b'"',
825-
None,
826-
None,
827-
false,
828-
file_compression_type.to_owned(),
829-
);
819+
let csv = CsvExec::builder(config)
820+
.with_has_header(true)
821+
.with_delimeter(b',')
822+
.with_quote(b'"')
823+
.with_escape(None)
824+
.with_comment(None)
825+
.with_newlines_in_values(false)
826+
.with_file_compression_type(file_compression_type.to_owned())
827+
.build();
830828
assert_eq!(13, csv.base_config.file_schema.fields().len());
831829
assert_eq!(3, csv.schema().fields().len());
832830

@@ -887,16 +885,15 @@ mod tests {
887885
let mut config = partitioned_csv_config(file_schema, file_groups);
888886
config.limit = Some(5);
889887

890-
let csv = CsvExec::new(
891-
config,
892-
true,
893-
b',',
894-
b'"',
895-
None,
896-
None,
897-
false,
898-
file_compression_type.to_owned(),
899-
);
888+
let csv = CsvExec::builder(config)
889+
.with_has_header(true)
890+
.with_delimeter(b',')
891+
.with_quote(b'"')
892+
.with_escape(None)
893+
.with_comment(None)
894+
.with_newlines_in_values(false)
895+
.with_file_compression_type(file_compression_type.to_owned())
896+
.build();
900897
assert_eq!(13, csv.base_config.file_schema.fields().len());
901898
assert_eq!(13, csv.schema().fields().len());
902899

@@ -954,16 +951,15 @@ mod tests {
954951
let mut config = partitioned_csv_config(file_schema, file_groups);
955952
config.limit = Some(5);
956953

957-
let csv = CsvExec::new(
958-
config,
959-
true,
960-
b',',
961-
b'"',
962-
None,
963-
None,
964-
false,
965-
file_compression_type.to_owned(),
966-
);
954+
let csv = CsvExec::builder(config)
955+
.with_has_header(true)
956+
.with_delimeter(b',')
957+
.with_quote(b'"')
958+
.with_escape(None)
959+
.with_comment(None)
960+
.with_newlines_in_values(false)
961+
.with_file_compression_type(file_compression_type.to_owned())
962+
.build();
967963
assert_eq!(14, csv.base_config.file_schema.fields().len());
968964
assert_eq!(14, csv.schema().fields().len());
969965

@@ -1020,16 +1016,15 @@ mod tests {
10201016

10211017
// we don't have `/date=xx/` in the path but that is ok because
10221018
// partitions are resolved during scan anyway
1023-
let csv = CsvExec::new(
1024-
config,
1025-
true,
1026-
b',',
1027-
b'"',
1028-
None,
1029-
None,
1030-
false,
1031-
file_compression_type.to_owned(),
1032-
);
1019+
let csv = CsvExec::builder(config)
1020+
.with_has_header(true)
1021+
.with_delimeter(b',')
1022+
.with_quote(b'"')
1023+
.with_escape(None)
1024+
.with_comment(None)
1025+
.with_newlines_in_values(false)
1026+
.with_file_compression_type(file_compression_type.to_owned())
1027+
.build();
10331028
assert_eq!(13, csv.base_config.file_schema.fields().len());
10341029
assert_eq!(2, csv.schema().fields().len());
10351030

@@ -1116,16 +1111,15 @@ mod tests {
11161111
.unwrap();
11171112

11181113
let config = partitioned_csv_config(file_schema, file_groups);
1119-
let csv = CsvExec::new(
1120-
config,
1121-
true,
1122-
b',',
1123-
b'"',
1124-
None,
1125-
None,
1126-
false,
1127-
file_compression_type.to_owned(),
1128-
);
1114+
let csv = CsvExec::builder(config)
1115+
.with_has_header(true)
1116+
.with_delimeter(b',')
1117+
.with_quote(b'"')
1118+
.with_escape(None)
1119+
.with_comment(None)
1120+
.with_newlines_in_values(false)
1121+
.with_file_compression_type(file_compression_type.to_owned())
1122+
.build();
11291123

11301124
let it = csv.execute(0, task_ctx).unwrap();
11311125
let batches: Vec<_> = it.try_collect().await.unwrap();

‎datafusion/core/src/physical_optimizer/enforce_distribution.rs‎

Lines changed: 49 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1463,18 +1463,21 @@ pub(crate) mod tests {
14631463
}
14641464

14651465
fn csv_exec_with_sort(output_ordering: Vec<Vec<PhysicalSortExpr>>) -> Arc<CsvExec> {
1466-
Arc::new(CsvExec::new(
1467-
FileScanConfig::new(ObjectStoreUrl::parse("test:///").unwrap(), schema())
1468-
.with_file(PartitionedFile::new("x".to_string(), 100))
1469-
.with_output_ordering(output_ordering),
1470-
false,
1471-
b',',
1472-
b'"',
1473-
None,
1474-
None,
1475-
false,
1476-
FileCompressionType::UNCOMPRESSED,
1477-
))
1466+
Arc::new(
1467+
CsvExec::builder(
1468+
FileScanConfig::new(ObjectStoreUrl::parse("test:///").unwrap(), schema())
1469+
.with_file(PartitionedFile::new("x".to_string(), 100))
1470+
.with_output_ordering(output_ordering),
1471+
)
1472+
.with_has_header(false)
1473+
.with_delimeter(b',')
1474+
.with_quote(b'"')
1475+
.with_escape(None)
1476+
.with_comment(None)
1477+
.with_newlines_in_values(false)
1478+
.with_file_compression_type(FileCompressionType::UNCOMPRESSED)
1479+
.build(),
1480+
)
14781481
}
14791482

14801483
fn csv_exec_multiple() -> Arc<CsvExec> {
@@ -1485,21 +1488,24 @@ pub(crate) mod tests {
14851488
fn csv_exec_multiple_sorted(
14861489
output_ordering: Vec<Vec<PhysicalSortExpr>>,
14871490
) -> Arc<CsvExec> {
1488-
Arc::new(CsvExec::new(
1489-
FileScanConfig::new(ObjectStoreUrl::parse("test:///").unwrap(), schema())
1490-
.with_file_groups(vec![
1491-
vec![PartitionedFile::new("x".to_string(), 100)],
1492-
vec![PartitionedFile::new("y".to_string(), 100)],
1493-
])
1494-
.with_output_ordering(output_ordering),
1495-
false,
1496-
b',',
1497-
b'"',
1498-
None,
1499-
None,
1500-
false,
1501-
FileCompressionType::UNCOMPRESSED,
1502-
))
1491+
Arc::new(
1492+
CsvExec::builder(
1493+
FileScanConfig::new(ObjectStoreUrl::parse("test:///").unwrap(), schema())
1494+
.with_file_groups(vec![
1495+
vec![PartitionedFile::new("x".to_string(), 100)],
1496+
vec![PartitionedFile::new("y".to_string(), 100)],
1497+
])
1498+
.with_output_ordering(output_ordering),
1499+
)
1500+
.with_has_header(false)
1501+
.with_delimeter(b',')
1502+
.with_quote(b'"')
1503+
.with_escape(None)
1504+
.with_comment(None)
1505+
.with_newlines_in_values(false)
1506+
.with_file_compression_type(FileCompressionType::UNCOMPRESSED)
1507+
.build(),
1508+
)
15031509
}
15041510

15051511
fn projection_exec_with_alias(
@@ -3761,20 +3767,23 @@ pub(crate) mod tests {
37613767
};
37623768

37633769
let plan = aggregate_exec_with_alias(
3764-
Arc::new(CsvExec::new(
3765-
FileScanConfig::new(
3766-
ObjectStoreUrl::parse("test:///").unwrap(),
3767-
schema(),
3770+
Arc::new(
3771+
CsvExec::builder(
3772+
FileScanConfig::new(
3773+
ObjectStoreUrl::parse("test:///").unwrap(),
3774+
schema(),
3775+
)
3776+
.with_file(PartitionedFile::new("x".to_string(), 100)),
37683777
)
3769-
.with_file(PartitionedFile::new("x".to_string(), 100)),
3770-
false,
3771-
b',',
3772-
b'"',
3773-
None,
3774-
None,
3775-
false,
3776-
compression_type,
3777-
)),
3778+
.with_has_header(false)
3779+
.with_delimeter(b',')
3780+
.with_quote(b'"')
3781+
.with_escape(None)
3782+
.with_comment(None)
3783+
.with_newlines_in_values(false)
3784+
.with_file_compression_type(compression_type)
3785+
.build(),
3786+
),
37783787
vec![("a".to_string(), "a".to_string())],
37793788
);
37803789
assert_optimized!(expected, plan, true, false, 2, true, 10, false);

0 commit comments

Comments
 (0)