Skip to content

Commit

Permalink
[Fix](orc-reader) Fix orc complex types when late materialization was…
Browse files Browse the repository at this point in the history
… turned on by disabling late materialization in this case. (apache#26548)

Fix orc complex types when late materialization was turned on in orc reader by disabling late materialization in this case.
  • Loading branch information
kaka11chen authored and 胥剑旭 committed Dec 14, 2023
1 parent 86ea061 commit 7d03585
Show file tree
Hide file tree
Showing 4 changed files with 25 additions and 1 deletion.
10 changes: 9 additions & 1 deletion be/src/vec/exec/format/orc/vorc_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -734,6 +734,13 @@ Status OrcReader::set_fill_columns(
}
}

for (auto& each : _tuple_descriptor->slots()) {
PrimitiveType column_type = each->col_type();
if (column_type == TYPE_ARRAY || column_type == TYPE_MAP || column_type == TYPE_STRUCT) {
_has_complex_type = true;
}
}

for (auto& kv : partition_columns) {
auto iter = predicate_columns.find(kv.first);
if (iter == predicate_columns.end()) {
Expand All @@ -754,7 +761,8 @@ Status OrcReader::set_fill_columns(
}
}

if (_enable_lazy_mat && _lazy_read_ctx.predicate_columns.first.size() > 0 &&
if (!_has_complex_type && _enable_lazy_mat &&
_lazy_read_ctx.predicate_columns.first.size() > 0 &&
_lazy_read_ctx.lazy_read_columns.size() > 0) {
_lazy_read_ctx.can_lazy_read = true;
}
Expand Down
1 change: 1 addition & 0 deletions be/src/vec/exec/format/orc/vorc_reader.h
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,7 @@ class OrcReader : public GenericReader {
std::shared_ptr<ObjectPool> _obj_pool;
std::unique_ptr<orc::StringDictFilter> _string_dict_filter;
bool _is_dict_cols_converted;
bool _has_complex_type = false;
};

class ORCFileInputStream : public orc::InputStream {
Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,9 @@ suite("test_hive_basic_type", "external_docker,hive,external_docker_hive,p0,exte
// hive tables in rcbinary format are not supported
//order_qt_37 """select * from ${catalog_name}.${ex_db_name}.rcbinary_all_types limit 1;"""

// orc_all_types_t predicate test
order_qt_41 """select * from ${catalog_name}.${ex_db_name}.orc_all_types_t where t_int = 3;"""

//sql """drop catalog if exists ${catalog_name} """
}
}
Expand Down

0 comments on commit 7d03585

Please sign in to comment.