Skip to content

Commit 7fbab21

Browse files
rui-moglutenperfbot
authored andcommitted
[13620] fix: Add config for requested type check in ReaderBase::convertType
Alchemy-item: [[13620] fix: Add config for requested type check in ReaderBase::conve…](#855 (comment)) commit 1/1 - 924fc2e
1 parent 59f2ffb commit 7fbab21

File tree

7 files changed

+176
-95
lines changed

7 files changed

+176
-95
lines changed

velox/connectors/hive/HiveConfig.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,13 @@ bool HiveConfig::readStatsBasedFilterReorderDisabled(
235235
config_->get<bool>(kReadStatsBasedFilterReorderDisabled, false));
236236
}
237237

238+
bool HiveConfig::isRequestedTypeCheckEnabled(
239+
const config::ConfigBase* session) const {
240+
return session->get<bool>(
241+
kEnableRequestedTypeCheckSession,
242+
config_->get<bool>(kEnableRequestedTypeCheck, true));
243+
}
244+
238245
std::string HiveConfig::hiveLocalDataPath() const {
239246
return config_->get<std::string>(kLocalDataPath, "");
240247
}

velox/connectors/hive/HiveConfig.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,11 @@ class HiveConfig {
197197
static constexpr const char* kPreserveFlatMapsInMemorySession =
198198
"hive.preserve_flat_maps_in_memory";
199199

200+
static constexpr const char* kEnableRequestedTypeCheck =
201+
"enable-requested-type-check";
202+
static constexpr const char* kEnableRequestedTypeCheckSession =
203+
"enable_requested_type_check";
204+
200205
InsertExistingPartitionsBehavior insertExistingPartitionsBehavior(
201206
const config::ConfigBase* session) const;
202207

@@ -270,6 +275,10 @@ class HiveConfig {
270275
bool readStatsBasedFilterReorderDisabled(
271276
const config::ConfigBase* session) const;
272277

278+
/// Whether to enable requested type check in the ReaderBase::convertType.
279+
/// Returns true by default.
280+
bool isRequestedTypeCheckEnabled(const config::ConfigBase* session) const;
281+
273282
/// Returns the file system path containing local data. If non-empty,
274283
/// initializes LocalHiveConnectorMetadata to provide metadata for the tables
275284
/// in the directory.

velox/connectors/hive/HiveConnectorUtil.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -604,6 +604,8 @@ void configureReaderOptions(
604604
}
605605

606606
readerOptions.setFileFormat(hiveSplit->fileFormat);
607+
readerOptions.setEnableRequestedTypeCheck(
608+
hiveConfig->isRequestedTypeCheckEnabled(sessionProperties));
607609
}
608610
}
609611

velox/docs/configs.rst

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -685,7 +685,11 @@ Each query can override the config by setting corresponding query session proper
685685
- bool
686686
- false
687687
- Whether to preserve flat maps in memory as FlatMapVectors instead of converting them to MapVectors. This is only applied during data reading inside the DWRF and Nimble readers, not during downstream processing like expression evaluation etc.
688-
688+
* - enable-requested-type-check
689+
- enable_requested_type_check
690+
- bool
691+
- true
692+
- Whether to enable requested type check in the `ReaderBase::convertType`. True by default.
689693

690694
``ORC File Format Configuration``
691695
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

velox/dwio/common/Options.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -638,6 +638,10 @@ class ReaderOptions : public io::ReaderOptions {
638638
return randomSkip_;
639639
}
640640

641+
bool enableRequestedTypeCheck() const {
642+
return enableRequestedTypeCheck_;
643+
}
644+
641645
void setRandomSkip(std::shared_ptr<random::RandomSkipTracker> randomSkip) {
642646
randomSkip_ = std::move(randomSkip);
643647
}
@@ -674,6 +678,10 @@ class ReaderOptions : public io::ReaderOptions {
674678
allowEmptyFile_ = value;
675679
}
676680

681+
void setEnableRequestedTypeCheck(bool enableRequestedTypeCheck) {
682+
enableRequestedTypeCheck_ = enableRequestedTypeCheck;
683+
}
684+
677685
private:
678686
uint64_t tailLocation_;
679687
FileFormat fileFormat_;
@@ -691,6 +699,7 @@ class ReaderOptions : public io::ReaderOptions {
691699
bool adjustTimestampToTimezone_{false};
692700
bool selectiveNimbleReaderEnabled_{false};
693701
bool allowEmptyFile_{false};
702+
bool enableRequestedTypeCheck_{true};
694703
};
695704

696705
struct WriterOptions {

0 commit comments

Comments
 (0)