Skip to content

Commit 8b1d26f

Browse files
xudong963adriangb
authored andcommitted
Support utf8view for spark hex (apache#16885)
1 parent cd5f632 commit 8b1d26f

File tree

2 files changed

+23
-0
lines changed
  • datafusion
    • spark/src/function/math
    • sqllogictest/test_files/spark/math

2 files changed

+23
-0
lines changed

datafusion/spark/src/function/math/hex.rs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ use arrow::{
2727
array::{as_dictionary_array, as_largestring_array, as_string_array},
2828
datatypes::Int32Type,
2929
};
30+
use datafusion_common::cast::as_string_view_array;
3031
use datafusion_common::{
3132
cast::{as_binary_array, as_fixed_size_binary_array, as_int64_array},
3233
exec_err, DataFusionError,
@@ -98,12 +99,14 @@ impl ScalarUDFImpl for SparkHex {
9899
match &arg_types[0] {
99100
DataType::Int64
100101
| DataType::Utf8
102+
| DataType::Utf8View
101103
| DataType::LargeUtf8
102104
| DataType::Binary
103105
| DataType::LargeBinary => Ok(vec![arg_types[0].clone()]),
104106
DataType::Dictionary(key_type, value_type) => match value_type.as_ref() {
105107
DataType::Int64
106108
| DataType::Utf8
109+
| DataType::Utf8View
107110
| DataType::LargeUtf8
108111
| DataType::Binary
109112
| DataType::LargeBinary => Ok(vec![arg_types[0].clone()]),
@@ -212,6 +215,16 @@ pub fn compute_hex(
212215

213216
Ok(ColumnarValue::Array(Arc::new(hexed)))
214217
}
218+
DataType::Utf8View => {
219+
let array = as_string_view_array(array)?;
220+
221+
let hexed: StringArray = array
222+
.iter()
223+
.map(|v| v.map(|b| hex_bytes(b, lowercase)).transpose())
224+
.collect::<Result<_, _>>()?;
225+
226+
Ok(ColumnarValue::Array(Arc::new(hexed)))
227+
}
215228
DataType::LargeUtf8 => {
216229
let array = as_largestring_array(array);
217230

datafusion/sqllogictest/test_files/spark/math/hex.slt

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,13 @@ SELECT hex(a) from VALUES ('foo'), (NULL), ('foobarbaz') AS t(a);
3838
666F6F
3939
NULL
4040
666F6F62617262617A
41+
42+
statement ok
43+
CREATE TABLE t_utf8view as VALUES (arrow_cast('foo', 'Utf8View')), (NULL), (arrow_cast('foobarbaz', 'Utf8View'));
44+
45+
query T
46+
SELECT hex(column1) FROM t_utf8view;
47+
----
48+
666F6F
49+
NULL
50+
666F6F62617262617A

0 commit comments

Comments
 (0)