From 52caf55843ef667309922dc80648cbf325490117 Mon Sep 17 00:00:00 2001 From: leorishdu <18771113323@163.com> Date: Thu, 24 Aug 2023 14:19:19 +0800 Subject: [PATCH] [Feature] Support light schema change of adding and dropping columns (#26246) Fixes #24341 This PR enables light schema change to be supported in --------- Signed-off-by: zhangqiang Co-authored-by: jijundu Co-authored-by: zhangqiang --- be/src/connector/binlog_connector.cpp | 4 +- be/src/connector/lake_connector.cpp | 6 +- be/src/exec/olap_scan_node.cpp | 2 +- .../exec/pipeline/scan/olap_chunk_source.cpp | 33 +- be/src/exec/pipeline/scan/olap_chunk_source.h | 1 + be/src/exec/tablet_info.cpp | 14 + be/src/exec/tablet_info.h | 2 + be/src/exec/tablet_scanner.cpp | 27 +- be/src/exec/tablet_scanner.h | 1 + be/src/runtime/descriptors.cpp | 3 + be/src/runtime/descriptors.h | 3 + be/src/runtime/local_tablets_channel.cpp | 1 + be/src/script/script.cpp | 4 +- be/src/storage/base_tablet.h | 21 +- be/src/storage/chunk_helper.cpp | 36 +- be/src/storage/chunk_helper.h | 13 +- be/src/storage/compaction.cpp | 39 +- be/src/storage/compaction.h | 6 +- be/src/storage/compaction_utils.cpp | 5 +- be/src/storage/compaction_utils.h | 4 +- be/src/storage/data_dir.cpp | 23 +- be/src/storage/delta_writer.cpp | 58 ++- be/src/storage/delta_writer.h | 9 +- be/src/storage/horizontal_compaction_task.cpp | 3 +- be/src/storage/lake/delta_writer.cpp | 6 +- be/src/storage/lake/general_tablet_writer.cpp | 4 +- .../lake/horizontal_compaction_task.cpp | 4 +- .../lake/lake_local_persistent_index.cpp | 4 +- be/src/storage/lake/lake_primary_index.cpp | 4 +- be/src/storage/lake/rowset_update_state.cpp | 53 +- be/src/storage/lake/rowset_update_state.h | 13 +- be/src/storage/lake/schema_change.cpp | 12 +- be/src/storage/lake/spark_load.cpp | 4 +- be/src/storage/lake/tablet_reader.cpp | 4 +- be/src/storage/lake/tablet_writer.h | 4 +- .../storage/lake/update_compaction_state.cpp | 6 +- be/src/storage/lake/update_compaction_state.h | 5 +- be/src/storage/lake/update_manager.cpp | 25 +- be/src/storage/lake/update_manager.h | 6 +- .../storage/lake/vertical_compaction_task.cpp | 4 +- be/src/storage/local_tablet_reader.cpp | 18 +- be/src/storage/memtable.cpp | 5 +- be/src/storage/memtable.h | 3 +- be/src/storage/meta_reader.cpp | 3 +- be/src/storage/meta_reader.h | 2 + be/src/storage/metadata_util.cpp | 19 +- be/src/storage/metadata_util.h | 16 +- be/src/storage/olap_meta_reader.cpp | 3 +- be/src/storage/persistent_index.cpp | 8 +- be/src/storage/predicate_parser.cpp | 28 +- be/src/storage/predicate_parser.h | 6 +- be/src/storage/primary_index.cpp | 8 +- be/src/storage/push_handler.cpp | 36 +- be/src/storage/push_handler.h | 6 +- .../horizontal_update_rowset_writer.cpp | 2 +- be/src/storage/rowset/rowset.cpp | 9 +- be/src/storage/rowset/rowset.h | 13 +- be/src/storage/rowset/rowset_factory.cpp | 2 +- be/src/storage/rowset/rowset_factory.h | 2 +- be/src/storage/rowset/rowset_meta.h | 22 +- be/src/storage/rowset/rowset_options.h | 3 +- be/src/storage/rowset/rowset_writer.cpp | 16 +- be/src/storage/rowset/rowset_writer_context.h | 2 +- be/src/storage/rowset/segment.cpp | 65 ++- be/src/storage/rowset/segment.h | 39 +- .../rowset/segment_chunk_iterator_adapter.cpp | 6 +- .../rowset/segment_chunk_iterator_adapter.h | 7 +- be/src/storage/rowset/segment_iterator.cpp | 9 +- be/src/storage/rowset/segment_options.cpp | 2 +- be/src/storage/rowset/segment_options.h | 3 + be/src/storage/rowset/segment_rewriter.cpp | 39 +- be/src/storage/rowset/segment_rewriter.h | 13 +- be/src/storage/rowset/segment_writer.cpp | 9 +- be/src/storage/rowset/segment_writer.h | 5 +- be/src/storage/rowset_column_update_state.cpp | 34 +- be/src/storage/rowset_column_update_state.h | 16 +- be/src/storage/rowset_merger.cpp | 25 +- be/src/storage/rowset_merger.h | 3 +- be/src/storage/rowset_update_state.cpp | 35 +- be/src/storage/rowset_update_state.h | 4 +- be/src/storage/schema_change.cpp | 57 ++- be/src/storage/schema_change.h | 13 +- be/src/storage/schema_change_utils.cpp | 53 +- be/src/storage/schema_change_utils.h | 4 +- be/src/storage/seek_tuple.h | 3 +- be/src/storage/snapshot_manager.cpp | 8 +- be/src/storage/snapshot_manager.h | 2 +- be/src/storage/tablet.cpp | 43 +- be/src/storage/tablet.h | 24 +- be/src/storage/tablet_manager.cpp | 10 +- be/src/storage/tablet_meta.cpp | 11 +- be/src/storage/tablet_meta.h | 8 +- be/src/storage/tablet_reader.cpp | 58 ++- be/src/storage/tablet_reader.h | 10 +- be/src/storage/tablet_schema.cpp | 136 ++++- be/src/storage/tablet_schema.h | 32 +- be/src/storage/tablet_schema_map.cpp | 5 +- be/src/storage/tablet_schema_map.h | 2 +- be/src/storage/tablet_updates.cpp | 55 ++- be/src/storage/task/engine_checksum_task.cpp | 6 +- be/src/storage/task/engine_clone_task.cpp | 2 +- be/src/storage/update_compaction_state.cpp | 4 +- be/src/storage/vertical_compaction_task.cpp | 6 +- be/src/tools/meta_tool.cpp | 6 +- be/test/runtime/lake_tablets_channel_test.cpp | 4 +- be/test/runtime/load_channel_test.cpp | 4 +- be/test/storage/base_compaction_test.cpp | 6 +- be/test/storage/binlog_manager_test.cpp | 6 +- be/test/storage/binlog_reader_test.cpp | 2 +- .../storage/conjunctive_predicates_test.cpp | 2 +- .../storage/cumulative_compaction_test.cpp | 8 +- .../default_compaction_policy_test.cpp | 6 +- be/test/storage/delete_handler_test.cpp | 114 +++-- be/test/storage/get_use_pk_index_test.cpp | 4 +- .../storage/lake/async_delta_writer_test.cpp | 6 +- .../auto_increment_partial_update_test.cpp | 4 +- be/test/storage/lake/compaction_task_test.cpp | 8 +- .../storage/lake/condition_update_test.cpp | 4 +- be/test/storage/lake/delta_writer_test.cpp | 6 +- be/test/storage/lake/partial_update_test.cpp | 4 +- .../lake/primary_key_compaction_task_test.cpp | 2 +- .../storage/lake/primary_key_publish_test.cpp | 2 +- be/test/storage/lake/rowset_test.cpp | 2 +- be/test/storage/lake/schema_change_test.cpp | 12 +- be/test/storage/lake/tablet_reader_test.cpp | 8 +- be/test/storage/lake/tablet_writer_test.cpp | 14 +- .../storage/memtable_flush_executor_test.cpp | 4 +- be/test/storage/memtable_test.cpp | 6 +- be/test/storage/persistent_index_test.cpp | 2 +- .../storage/publish_version_manager_test.cpp | 2 +- be/test/storage/publish_version_task_test.cpp | 9 +- .../rowset/column_reader_writer_test.cpp | 2 +- be/test/storage/rowset/map_column_rw_test.cpp | 2 +- be/test/storage/rowset/rowset_test.cpp | 52 +- .../storage/rowset/segment_iterator_test.cpp | 20 +- .../storage/rowset/segment_rewriter_test.cpp | 20 +- be/test/storage/rowset/segment_test.cpp | 56 +-- .../storage/rowset/struct_column_rw_test.cpp | 2 +- .../rowset_column_partial_update_test.cpp | 8 +- .../rowset_column_update_state_test.cpp | 6 +- be/test/storage/rowset_merger_test.cpp | 2 +- be/test/storage/rowset_update_state_test.cpp | 8 +- be/test/storage/schema_change_test.cpp | 22 +- .../size_tiered_compaction_policy_test.cpp | 6 +- be/test/storage/table_reader_remote_test.cpp | 2 +- be/test/storage/table_reader_test.cpp | 2 +- be/test/storage/tablet_binlog_test.cpp | 2 +- be/test/storage/tablet_mgr_test.cpp | 10 +- be/test/storage/tablet_updates_test.cpp | 90 ++-- .../engine_storage_migration_task_test.cpp | 11 +- be/test/storage/update_manager_test.cpp | 2 +- .../java/com/starrocks/alter/AlterJobV2.java | 8 + .../alter/MaterializedViewHandler.java | 3 + .../java/com/starrocks/alter/RollupJobV2.java | 4 - .../starrocks/alter/SchemaChangeHandler.java | 467 +++++++++++++++--- .../starrocks/alter/SchemaChangeJobV2.java | 17 +- .../starrocks/analysis/SlotDescriptor.java | 9 + .../java/com/starrocks/catalog/Column.java | 49 +- .../catalog/MaterializedIndexMeta.java | 8 + .../java/com/starrocks/catalog/OlapTable.java | 37 ++ .../com/starrocks/catalog/TableProperty.java | 13 + .../catalog/TabletInvertedIndex.java | 4 + .../java/com/starrocks/common/Config.java | 6 + .../common/util/PropertyAnalyzer.java | 28 ++ .../com/starrocks/journal/JournalEntity.java | 5 + .../com/starrocks/load/OlapDeleteJob.java | 10 +- .../starrocks/load/loadv2/SparkLoadJob.java | 18 +- .../java/com/starrocks/persist/EditLog.java | 9 + .../com/starrocks/persist/OperationType.java | 3 + .../persist/TableAddOrDropColumnsInfo.java | 139 ++++++ .../com/starrocks/planner/OlapScanNode.java | 13 +- .../com/starrocks/planner/OlapTableSink.java | 11 +- .../com/starrocks/server/GlobalStateMgr.java | 6 + .../starrocks/server/OlapTableFactory.java | 20 + .../java/com/starrocks/sql/ast/ColumnDef.java | 3 +- .../com/starrocks/task/AlterReplicaTask.java | 38 +- .../java/com/starrocks/task/PushTask.java | 12 +- .../alter/SchemaChangeHandlerTest.java | 397 +++++++++++++-- .../com/starrocks/catalog/ColumnTest.java | 25 + .../common/PropertyAnalyzerTest.java | 7 + .../TableAddOrDropColumnsInfoTest.java | 43 ++ .../starrocks/task/AlterReplicaTaskTest.java | 3 +- .../starrocks/utframe/TestWithFeService.java | 228 +++++++++ gensrc/proto/descriptors.proto | 2 + gensrc/proto/olap_file.proto | 3 +- gensrc/proto/tablet_schema.proto | 1 + gensrc/thrift/AgentService.thrift | 2 + gensrc/thrift/Descriptors.thrift | 6 +- gensrc/thrift/PlanNodes.thrift | 2 + .../R/test_light_schema_change | 85 ++++ .../T/test_light_schema_change | 30 ++ 191 files changed, 2969 insertions(+), 961 deletions(-) create mode 100644 fe/fe-core/src/main/java/com/starrocks/persist/TableAddOrDropColumnsInfo.java create mode 100644 fe/fe-core/src/test/java/com/starrocks/persist/TableAddOrDropColumnsInfoTest.java create mode 100644 fe/fe-core/src/test/java/com/starrocks/utframe/TestWithFeService.java create mode 100644 test/sql/test_light_schema_change/R/test_light_schema_change create mode 100644 test/sql/test_light_schema_change/T/test_light_schema_change diff --git a/be/src/connector/binlog_connector.cpp b/be/src/connector/binlog_connector.cpp index b52a0dd0b6fa5..2a5d318df55d0 100644 --- a/be/src/connector/binlog_connector.cpp +++ b/be/src/connector/binlog_connector.cpp @@ -177,7 +177,7 @@ BinlogMetaFieldMap BinlogDataSource::_build_binlog_meta_fields(ColumnId start_ci } StatusOr BinlogDataSource::_build_binlog_schema() { - BinlogMetaFieldMap binlog_meta_map = _build_binlog_meta_fields(_tablet->tablet_schema().num_columns()); + BinlogMetaFieldMap binlog_meta_map = _build_binlog_meta_fields(_tablet->tablet_schema()->num_columns()); std::vector data_column_cids; std::vector meta_column_slot_index; Fields meta_fields; @@ -211,7 +211,7 @@ StatusOr BinlogDataSource::_build_binlog_schema() { return Status::InternalError("failed to build binlog schema, no materialized data slot!"); } - const TabletSchema& tablet_schema = _tablet->tablet_schema(); + const TabletSchemaCSPtr& tablet_schema = _tablet->tablet_schema(); Schema schema = ChunkHelper::convert_schema(tablet_schema, data_column_cids); for (int32_t i = 0; i < meta_column_slot_index.size(); i++) { uint32_t index = meta_column_slot_index[i]; diff --git a/be/src/connector/lake_connector.cpp b/be/src/connector/lake_connector.cpp index 9f2ca603a4030..b19c980742115 100644 --- a/be/src/connector/lake_connector.cpp +++ b/be/src/connector/lake_connector.cpp @@ -383,7 +383,7 @@ Status LakeDataSource::init_reader_params(const std::vector& key std::vector& reader_columns) { const TLakeScanNode& thrift_lake_scan_node = _provider->_t_lake_scan_node; bool skip_aggregation = thrift_lake_scan_node.is_preaggregation; - auto parser = _obj_pool.add(new PredicateParser(*_tablet_schema)); + auto parser = _obj_pool.add(new PredicateParser(_tablet_schema)); _params.is_pipeline = true; _params.reader_type = READER_QUERY; _params.skip_aggregation = skip_aggregation; @@ -458,14 +458,14 @@ Status LakeDataSource::init_tablet_reader(RuntimeState* runtime_state) { RETURN_IF_ERROR(init_unused_output_columns(thrift_lake_scan_node.unused_output_column_name)); RETURN_IF_ERROR(init_scanner_columns(scanner_columns)); RETURN_IF_ERROR(init_reader_params(_scanner_ranges, scanner_columns, reader_columns)); - starrocks::Schema child_schema = ChunkHelper::convert_schema(*_tablet_schema, reader_columns); + starrocks::Schema child_schema = ChunkHelper::convert_schema(_tablet_schema, reader_columns); ASSIGN_OR_RETURN(auto tablet, ExecEnv::GetInstance()->lake_tablet_manager()->get_tablet(_scan_range.tablet_id)); ASSIGN_OR_RETURN(_reader, tablet.new_reader(_version, std::move(child_schema))); if (reader_columns.size() == scanner_columns.size()) { _prj_iter = _reader; } else { - starrocks::Schema output_schema = ChunkHelper::convert_schema(*_tablet_schema, scanner_columns); + starrocks::Schema output_schema = ChunkHelper::convert_schema(_tablet_schema, scanner_columns); _prj_iter = new_projection_iterator(output_schema, _reader); } diff --git a/be/src/exec/olap_scan_node.cpp b/be/src/exec/olap_scan_node.cpp index 2977b7ab55631..7bf2ffb61565a 100644 --- a/be/src/exec/olap_scan_node.cpp +++ b/be/src/exec/olap_scan_node.cpp @@ -459,7 +459,7 @@ StatusOr OlapScanNode::_could_tablet_internal_parallel( StatusOr OlapScanNode::_could_split_tablet_physically(const std::vector& scan_ranges) const { // Keys type needn't merge or aggregate. ASSIGN_OR_RETURN(TabletSharedPtr first_tablet, get_tablet(&(scan_ranges[0].scan_range.internal_scan_range))); - KeysType keys_type = first_tablet->tablet_schema().keys_type(); + KeysType keys_type = first_tablet->tablet_schema()->keys_type(); const auto skip_aggr = thrift_olap_scan_node().is_preaggregation; bool is_keys_type_matched = keys_type == PRIMARY_KEYS || keys_type == DUP_KEYS || ((keys_type == UNIQUE_KEYS || keys_type == AGG_KEYS) && skip_aggr); diff --git a/be/src/exec/pipeline/scan/olap_chunk_source.cpp b/be/src/exec/pipeline/scan/olap_chunk_source.cpp index edf7969eca768..8b63ecf9de71a 100644 --- a/be/src/exec/pipeline/scan/olap_chunk_source.cpp +++ b/be/src/exec/pipeline/scan/olap_chunk_source.cpp @@ -160,7 +160,7 @@ Status OlapChunkSource::_init_reader_params(const std::vector& reader_columns) { const TOlapScanNode& thrift_olap_scan_node = _scan_node->thrift_olap_scan_node(); bool skip_aggregation = thrift_olap_scan_node.is_preaggregation; - auto parser = _obj_pool.add(new PredicateParser(_tablet->tablet_schema())); + auto parser = _obj_pool.add(new PredicateParser(_tablet_schema)); _params.is_pipeline = true; _params.reader_type = READER_QUERY; _params.skip_aggregation = skip_aggregation; @@ -211,11 +211,11 @@ Status OlapChunkSource::_init_reader_params(const std::vectornum_key_columns(); i++) { + for (size_t i = 0; i < _tablet_schema->num_key_columns(); i++) { reader_columns.push_back(i); } for (auto index : scanner_columns) { - if (!_tablet->tablet_schema().column(index).is_key()) { + if (!_tablet_schema->column(index).is_key()) { reader_columns.push_back(index); } } @@ -229,7 +229,7 @@ Status OlapChunkSource::_init_reader_params(const std::vector& scanner_columns) { for (auto slot : *_slots) { DCHECK(slot->is_materialized()); - int32_t index = _tablet->field_index(slot->col_name()); + int32_t index = _tablet_schema->field_index(slot->col_name()); if (index < 0) { std::stringstream ss; ss << "invalid field name: " << slot->col_name(); @@ -253,7 +253,7 @@ Status OlapChunkSource::_init_scanner_columns(std::vector& scanner_col Status OlapChunkSource::_init_unused_output_columns(const std::vector& unused_output_columns) { for (const auto& col_name : unused_output_columns) { - int32_t index = _tablet->field_index(col_name); + int32_t index = _tablet_schema->field_index(col_name); if (index < 0) { std::stringstream ss; ss << "invalid field name: " << col_name; @@ -294,20 +294,33 @@ Status OlapChunkSource::_init_olap_reader(RuntimeState* runtime_state) { std::vector reader_columns; RETURN_IF_ERROR(_get_tablet(_scan_range)); + + auto tablet_schema_ptr = _tablet->tablet_schema(); + _tablet_schema = TabletSchema::copy(tablet_schema_ptr); + + // if column_desc come from fe, reset tablet schema + if (!_scan_node->thrift_olap_scan_node().columns_desc.empty() && + _scan_node->thrift_olap_scan_node().columns_desc[0].col_unique_id >= 0) { + _tablet_schema->clear_columns(); + for (const auto& column_desc : _scan_node->thrift_olap_scan_node().columns_desc) { + _tablet_schema->append_column(TabletColumn(column_desc)); + } + } + RETURN_IF_ERROR(_init_global_dicts(&_params)); RETURN_IF_ERROR(_init_unused_output_columns(thrift_olap_scan_node.unused_output_column_name)); RETURN_IF_ERROR(_init_scanner_columns(scanner_columns)); RETURN_IF_ERROR(_init_reader_params(_scan_ctx->key_ranges(), scanner_columns, reader_columns)); - const TabletSchema& tablet_schema = _tablet->tablet_schema(); - starrocks::Schema child_schema = ChunkHelper::convert_schema(tablet_schema, reader_columns); + + starrocks::Schema child_schema = ChunkHelper::convert_schema(_tablet_schema, reader_columns); RETURN_IF_ERROR(_init_column_access_paths(&child_schema)); _reader = std::make_shared(_tablet, Version(_morsel->from_version(), _version), - std::move(child_schema), _morsel->rowsets()); + std::move(child_schema), _morsel->rowsets(), &_tablet_schema); if (reader_columns.size() == scanner_columns.size()) { _prj_iter = _reader; } else { - starrocks::Schema output_schema = ChunkHelper::convert_schema(tablet_schema, scanner_columns); + starrocks::Schema output_schema = ChunkHelper::convert_schema(_tablet_schema, scanner_columns); _prj_iter = new_projection_iterator(output_schema, _reader); } @@ -349,7 +362,7 @@ Status OlapChunkSource::_init_global_dicts(TabletReaderParams* params) { auto iter = global_dict_map.find(slot->id()); if (iter != global_dict_map.end()) { auto& dict_map = iter->second.first; - int32_t index = _tablet->field_index(slot->col_name()); + int32_t index = _tablet_schema->field_index(slot->col_name()); DCHECK(index >= 0); global_dict->emplace(index, const_cast(&dict_map)); } diff --git a/be/src/exec/pipeline/scan/olap_chunk_source.h b/be/src/exec/pipeline/scan/olap_chunk_source.h index f55b241fba268..a850ca46fbd93 100644 --- a/be/src/exec/pipeline/scan/olap_chunk_source.h +++ b/be/src/exec/pipeline/scan/olap_chunk_source.h @@ -80,6 +80,7 @@ class OlapChunkSource final : public ChunkSource { ObjectPool _obj_pool; TabletSharedPtr _tablet; + std::shared_ptr _tablet_schema; int64_t _version = 0; RuntimeState* _runtime_state = nullptr; diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp index 79748db489be6..e65c03a4c145b 100644 --- a/be/src/exec/tablet_info.cpp +++ b/be/src/exec/tablet_info.cpp @@ -19,6 +19,7 @@ #include "column/column_helper.h" #include "exprs/expr.h" #include "runtime/mem_pool.h" +#include "storage/tablet_schema.h" #include "types/constexpr.h" #include "util/string_parser.hpp" @@ -46,6 +47,9 @@ void OlapTableIndexSchema::to_protobuf(POlapTableIndexSchema* pindex) const { for (auto slot : slots) { pindex->add_columns(slot->col_name()); } + for (auto column : columns) { + column->to_schema_pb(pindex->add_columns_desc()); + } } Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { @@ -69,6 +73,11 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { index->slots.emplace_back(it->second); } } + for (auto& pcolumn_desc : p_index.columns_desc()) { + TabletColumn* tc = _obj_pool.add(new TabletColumn()); + tc->init_from_pb(pcolumn_desc); + index->columns.emplace_back(tc); + } _indexes.emplace_back(index); } @@ -99,6 +108,11 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) { index->slots.emplace_back(it->second); } } + for (auto& tcolumn_desc : t_index.columns_desc) { + TabletColumn* tc = _obj_pool.add(new TabletColumn()); + tc->init_from_thrift(tcolumn_desc); + index->columns.emplace_back(tc); + } _indexes.emplace_back(index); } diff --git a/be/src/exec/tablet_info.h b/be/src/exec/tablet_info.h index 0bc6748e0307f..076675eb86dd0 100644 --- a/be/src/exec/tablet_info.h +++ b/be/src/exec/tablet_info.h @@ -25,6 +25,7 @@ #include "gen_cpp/Descriptors_types.h" #include "gen_cpp/descriptors.pb.h" #include "runtime/descriptors.h" +#include "storage/tablet_schema.h" #include "util/random.h" namespace starrocks { @@ -36,6 +37,7 @@ struct OlapTableIndexSchema { int64_t index_id; std::vector slots; int32_t schema_hash; + std::vector columns; void to_protobuf(POlapTableIndexSchema* pindex) const; }; diff --git a/be/src/exec/tablet_scanner.cpp b/be/src/exec/tablet_scanner.cpp index 8f6d57e575e77..92d4e28b72c3d 100644 --- a/be/src/exec/tablet_scanner.cpp +++ b/be/src/exec/tablet_scanner.cpp @@ -47,17 +47,28 @@ Status TabletScanner::init(RuntimeState* runtime_state, const TabletScannerParam RETURN_IF_ERROR(Expr::clone_if_not_exists(runtime_state, &_pool, *params.conjunct_ctxs, &_conjunct_ctxs)); RETURN_IF_ERROR(_get_tablet(params.scan_range)); + + auto tablet_schema_ptr = _tablet->tablet_schema(); + _tablet_schema = TabletSchema::copy(tablet_schema_ptr); + + // if column_desc come from fe, reset tablet schema + if (!_parent->_olap_scan_node.columns_desc.empty() && _parent->_olap_scan_node.columns_desc[0].col_unique_id >= 0) { + _tablet_schema->clear_columns(); + for (const auto& column_desc : _parent->_olap_scan_node.columns_desc) { + _tablet_schema->append_column(TabletColumn(column_desc)); + } + } + RETURN_IF_ERROR(_init_unused_output_columns(*params.unused_output_columns)); RETURN_IF_ERROR(_init_return_columns()); RETURN_IF_ERROR(_init_global_dicts()); RETURN_IF_ERROR(_init_reader_params(params.key_ranges)); - const TabletSchema& tablet_schema = _tablet->tablet_schema(); - Schema child_schema = ChunkHelper::convert_schema(tablet_schema, _reader_columns); + Schema child_schema = ChunkHelper::convert_schema(_tablet_schema, _reader_columns); _reader = std::make_shared(_tablet, Version(0, _version), std::move(child_schema)); if (_reader_columns.size() == _scanner_columns.size()) { _prj_iter = _reader; } else { - Schema output_schema = ChunkHelper::convert_schema(tablet_schema, _scanner_columns); + Schema output_schema = ChunkHelper::convert_schema(_tablet_schema, _scanner_columns); _prj_iter = new_projection_iterator(output_schema, _reader); } @@ -181,11 +192,11 @@ Status TabletScanner::_init_reader_params(const std::vector* key if (_skip_aggregation) { _reader_columns = _scanner_columns; } else { - for (size_t i = 0; i < _tablet->num_key_columns(); i++) { + for (size_t i = 0; i < _tablet_schema->num_key_columns(); i++) { _reader_columns.push_back(i); } for (auto index : _scanner_columns) { - if (!_tablet->tablet_schema().column(index).is_key()) { + if (!_tablet_schema->column(index).is_key()) { _reader_columns.push_back(index); } } @@ -202,7 +213,7 @@ Status TabletScanner::_init_return_columns() { if (!slot->is_materialized()) { continue; } - int32_t index = _tablet->field_index(slot->col_name()); + int32_t index = _tablet_schema->field_index(slot->col_name()); if (index < 0) { auto msg = strings::Substitute("Invalid column name: $0", slot->col_name()); LOG(WARNING) << msg; @@ -224,7 +235,7 @@ Status TabletScanner::_init_return_columns() { Status TabletScanner::_init_unused_output_columns(const std::vector& unused_output_columns) { for (const auto& col_name : unused_output_columns) { - int32_t index = _tablet->field_index(col_name); + int32_t index = _tablet_schema->field_index(col_name); if (index < 0) { auto msg = strings::Substitute("Invalid column name: $0", col_name); LOG(WARNING) << msg; @@ -248,7 +259,7 @@ Status TabletScanner::_init_global_dicts() { auto iter = global_dict_map.find(slot->id()); if (iter != global_dict_map.end()) { auto& dict_map = iter->second.first; - int32_t index = _tablet->field_index(slot->col_name()); + int32_t index = _tablet_schema->field_index(slot->col_name()); DCHECK(index >= 0); global_dict->emplace(index, const_cast(&dict_map)); } diff --git a/be/src/exec/tablet_scanner.h b/be/src/exec/tablet_scanner.h index f76a35a936cf7..f21b7a5ba7454 100644 --- a/be/src/exec/tablet_scanner.h +++ b/be/src/exec/tablet_scanner.h @@ -104,6 +104,7 @@ class TabletScanner { std::shared_ptr _reader; TabletSharedPtr _tablet; + TabletSchemaSPtr _tablet_schema; int64_t _version = 0; // output columns of `this` TabletScanner, i.e, the final output columns of `get_chunk`. diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp index 9364502800f57..81b1df6449cbd 100644 --- a/be/src/runtime/descriptors.cpp +++ b/be/src/runtime/descriptors.cpp @@ -65,6 +65,7 @@ SlotDescriptor::SlotDescriptor(SlotId id, std::string name, TypeDescriptor type) _parent(0), _null_indicator_offset(0, 0), _col_name(std::move(name)), + _col_unique_id(-1), _slot_idx(0), _slot_size(_type.get_slot_size()), _is_materialized(false), @@ -77,6 +78,7 @@ SlotDescriptor::SlotDescriptor(const TSlotDescriptor& tdesc) _parent(tdesc.parent), _null_indicator_offset(tdesc.nullIndicatorByte, tdesc.nullIndicatorBit), _col_name(tdesc.colName), + _col_unique_id(tdesc.col_unique_id), _slot_idx(tdesc.slotIdx), _slot_size(_type.get_slot_size()), _is_materialized(tdesc.isMaterialized), @@ -89,6 +91,7 @@ SlotDescriptor::SlotDescriptor(const PSlotDescriptor& pdesc) _parent(pdesc.parent()), _null_indicator_offset(pdesc.null_indicator_byte(), pdesc.null_indicator_bit()), _col_name(pdesc.col_name()), + _col_unique_id(-1), _slot_idx(pdesc.slot_idx()), _slot_size(_type.get_slot_size()), _is_materialized(pdesc.is_materialized()), diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h index 3c5dc4e1d772a..445ed9a0c2eb5 100644 --- a/be/src/runtime/descriptors.h +++ b/be/src/runtime/descriptors.h @@ -107,6 +107,8 @@ class SlotDescriptor { std::string debug_string() const; + int32_t col_unique_id() const { return _col_unique_id; } + private: friend class DescriptorTbl; friend class TupleDescriptor; @@ -119,6 +121,7 @@ class SlotDescriptor { const TupleId _parent; const NullIndicatorOffset _null_indicator_offset; const std::string _col_name; + const int32_t _col_unique_id; // the idx of the slot in the tuple descriptor (0-based). // this is provided by the FE diff --git a/be/src/runtime/local_tablets_channel.cpp b/be/src/runtime/local_tablets_channel.cpp index 8500ac09206f3..90873e5cdd822 100644 --- a/be/src/runtime/local_tablets_channel.cpp +++ b/be/src/runtime/local_tablets_channel.cpp @@ -514,6 +514,7 @@ Status LocalTabletsChannel::_open_all_writers(const PTabletWriterOpenRequest& pa options.timeout_ms = params.timeout_ms(); options.write_quorum = params.write_quorum(); options.miss_auto_increment_column = params.miss_auto_increment_column(); + options.ptable_schema_param = params.schema(); if (params.is_replicated_storage()) { for (auto& replica : tablet.replicas()) { options.replicas.emplace_back(replica); diff --git a/be/src/script/script.cpp b/be/src/script/script.cpp index bfe9dc39460b2..6795c97b2eca6 100644 --- a/be/src/script/script.cpp +++ b/be/src/script/script.cpp @@ -82,7 +82,7 @@ static std::string tablet_set_tablet_state(Tablet& tablet, int state) { } static const TabletSchema& tablet_tablet_schema(Tablet& tablet) { - return tablet.tablet_schema(); + return tablet.unsafe_tablet_schema_ref(); } static uint64_t tablet_tablet_id(Tablet& tablet) { @@ -421,7 +421,7 @@ class StorageEngineRef { { auto& cls = m.klass("Rowset"); REG_METHOD(Rowset, rowset_id_str); - REG_METHOD(Rowset, schema); + REG_METHOD(Rowset, schema_ref); REG_METHOD(Rowset, start_version); REG_METHOD(Rowset, end_version); REG_METHOD(Rowset, creation_time); diff --git a/be/src/storage/base_tablet.h b/be/src/storage/base_tablet.h index d12a58ab6cf07..016107fda4b5e 100644 --- a/be/src/storage/base_tablet.h +++ b/be/src/storage/base_tablet.h @@ -99,7 +99,20 @@ class BaseTablet : public std::enable_shared_from_this { bool equal(int64_t tablet_id, int32_t schema_hash); // properties encapsulated in TabletSchema - const TabletSchema& tablet_schema() const; + virtual const TabletSchema& unsafe_tablet_schema_ref() const; + + virtual const TabletSchemaCSPtr tablet_schema() const; + + bool set_tablet_schema_into_rowset_meta() { + bool flag = false; + for (const RowsetMetaSharedPtr& rowset_meta : _tablet_meta->all_rs_metas()) { + if (!rowset_meta->get_meta_pb().has_tablet_schema()) { + rowset_meta->set_tablet_schema(tablet_schema()); + flag = true; + } + } + return flag; + } protected: virtual void on_shutdown() {} @@ -172,8 +185,12 @@ inline bool BaseTablet::equal(int64_t id, int32_t hash) { return tablet_id() == id && schema_hash() == hash; } -inline const TabletSchema& BaseTablet::tablet_schema() const { +inline const TabletSchema& BaseTablet::unsafe_tablet_schema_ref() const { return _tablet_meta->tablet_schema(); } +inline const TabletSchemaCSPtr BaseTablet::tablet_schema() const { + return _tablet_meta->tablet_schema_ptr(); +} + } /* namespace starrocks */ diff --git a/be/src/storage/chunk_helper.cpp b/be/src/storage/chunk_helper.cpp index e0b733b805d8d..b4c336bd0b47e 100644 --- a/be/src/storage/chunk_helper.cpp +++ b/be/src/storage/chunk_helper.cpp @@ -144,37 +144,37 @@ Field ChunkHelper::convert_field(ColumnId id, const TabletColumn& c) { return f; } -starrocks::Schema ChunkHelper::convert_schema(const starrocks::TabletSchema& schema) { - return starrocks::Schema(schema.schema()); +starrocks::Schema ChunkHelper::convert_schema(const starrocks::TabletSchemaCSPtr& schema) { + return starrocks::Schema(schema->schema()); } -starrocks::Schema ChunkHelper::convert_schema(const starrocks::TabletSchema& schema, +starrocks::Schema ChunkHelper::convert_schema(const starrocks::TabletSchemaCSPtr& schema, const std::vector& cids) { - return starrocks::Schema(schema.schema(), cids); + return starrocks::Schema(schema->schema(), cids); } -starrocks::Schema ChunkHelper::get_short_key_schema(const starrocks::TabletSchema& schema) { +starrocks::Schema ChunkHelper::get_short_key_schema(const starrocks::TabletSchemaCSPtr& schema) { std::vector short_key_cids; - const auto& sort_key_idxes = schema.sort_key_idxes(); - short_key_cids.reserve(schema.num_short_key_columns()); - for (auto i = 0; i < schema.num_short_key_columns(); ++i) { + const auto& sort_key_idxes = schema->sort_key_idxes(); + short_key_cids.reserve(schema->num_short_key_columns()); + for (auto i = 0; i < schema->num_short_key_columns(); ++i) { short_key_cids.push_back(sort_key_idxes[i]); } - return starrocks::Schema(schema.schema(), short_key_cids); + return starrocks::Schema(schema->schema(), short_key_cids); } -starrocks::Schema ChunkHelper::get_sort_key_schema(const starrocks::TabletSchema& schema) { - std::vector sort_key_iota_idxes(schema.sort_key_idxes().size()); +starrocks::Schema ChunkHelper::get_sort_key_schema(const starrocks::TabletSchemaCSPtr& schema) { + std::vector sort_key_iota_idxes(schema->sort_key_idxes().size()); std::iota(sort_key_iota_idxes.begin(), sort_key_iota_idxes.end(), 0); - return starrocks::Schema(schema.schema(), schema.sort_key_idxes(), sort_key_iota_idxes); + return starrocks::Schema(schema->schema(), schema->sort_key_idxes(), sort_key_iota_idxes); } -starrocks::Schema ChunkHelper::get_sort_key_schema_by_primary_key(const starrocks::TabletSchema& tablet_schema) { - std::vector primary_key_iota_idxes(tablet_schema.num_key_columns()); +starrocks::Schema ChunkHelper::get_sort_key_schema_by_primary_key(const starrocks::TabletSchemaCSPtr& tablet_schema) { + std::vector primary_key_iota_idxes(tablet_schema->num_key_columns()); std::iota(primary_key_iota_idxes.begin(), primary_key_iota_idxes.end(), 0); - std::vector all_keys_iota_idxes(tablet_schema.num_columns()); + std::vector all_keys_iota_idxes(tablet_schema->num_columns()); std::iota(all_keys_iota_idxes.begin(), all_keys_iota_idxes.end(), 0); - return starrocks::Schema(tablet_schema.schema(), all_keys_iota_idxes, primary_key_iota_idxes); + return starrocks::Schema(tablet_schema->schema(), all_keys_iota_idxes, primary_key_iota_idxes); } ColumnId ChunkHelper::max_column_id(const starrocks::Schema& schema) { @@ -293,7 +293,7 @@ std::vector ChunkHelper::get_char_field_indexes(const Schema& schema) { } void ChunkHelper::padding_char_columns(const std::vector& char_column_indexes, const Schema& schema, - const starrocks::TabletSchema& tschema, Chunk* chunk) { + const starrocks::TabletSchemaCSPtr& tschema, Chunk* chunk) { size_t num_rows = chunk->num_rows(); for (auto field_index : char_column_indexes) { Column* column = chunk->get_column_by_index(field_index).get(); @@ -310,7 +310,7 @@ void ChunkHelper::padding_char_columns(const std::vector& char_column_in Bytes& new_bytes = new_binary->get_bytes(); // |schema| maybe partial columns in vertical compaction, so get char column length by name. - uint32_t len = tschema.column(tschema.field_index(schema.field(field_index)->name())).length(); + uint32_t len = tschema->column(tschema->field_index(schema.field(field_index)->name())).length(); new_offset.resize(num_rows + 1); new_bytes.assign(num_rows * len, 0); // padding 0 diff --git a/be/src/storage/chunk_helper.h b/be/src/storage/chunk_helper.h index c9ed1350cae8d..bd2c69640c743 100644 --- a/be/src/storage/chunk_helper.h +++ b/be/src/storage/chunk_helper.h @@ -20,6 +20,7 @@ #include "column/vectorized_fwd.h" #include "storage/olap_common.h" #include "storage/olap_type_infra.h" +#include "tablet_schema.h" namespace starrocks { @@ -36,19 +37,19 @@ class ChunkHelper { static Field convert_field(ColumnId id, const TabletColumn& c); // Convert TabletSchema to Schema with changing format v1 type to format v2 type. - static Schema convert_schema(const TabletSchema& schema); + static Schema convert_schema(const TabletSchemaCSPtr& schema); // Convert TabletSchema to Schema with changing format v1 type to format v2 type. - static Schema convert_schema(const TabletSchema& schema, const std::vector& cids); + static Schema convert_schema(const TabletSchemaCSPtr& schema, const std::vector& cids); // Get schema with format v2 type containing short key columns from TabletSchema. - static Schema get_short_key_schema(const TabletSchema& schema); + static Schema get_short_key_schema(const TabletSchemaCSPtr& schema); // Get schema with format v2 type containing sort key columns from TabletSchema. - static Schema get_sort_key_schema(const TabletSchema& schema); + static Schema get_sort_key_schema(const TabletSchemaCSPtr& schema); // Get schema with format v2 type containing sort key columns filled by primary key columns from TabletSchema. - static Schema get_sort_key_schema_by_primary_key(const starrocks::TabletSchema& tablet_schema); + static Schema get_sort_key_schema_by_primary_key(const starrocks::TabletSchemaCSPtr& tablet_schema); static ColumnId max_column_id(const Schema& schema); @@ -75,7 +76,7 @@ class ChunkHelper { // Padding char columns static void padding_char_columns(const std::vector& char_column_indexes, const Schema& schema, - const TabletSchema& tschema, Chunk* chunk); + const TabletSchemaCSPtr& tschema, Chunk* chunk); // Reorder columns of `chunk` according to the order of |tuple_desc|. static void reorder_chunk(const TupleDescriptor& tuple_desc, Chunk* chunk); diff --git a/be/src/storage/compaction.cpp b/be/src/storage/compaction.cpp index 541755f774ca3..bad37ee611f51 100644 --- a/be/src/storage/compaction.cpp +++ b/be/src/storage/compaction.cpp @@ -55,6 +55,12 @@ Status Compaction::do_compaction() { return st; } +RowsetSharedPtr& tablet_meta_with_max_rowset_version(std::vector rowsets) { + return *std::max_element(rowsets.begin(), rowsets.end(), [](const RowsetSharedPtr& a, const RowsetSharedPtr& b) { + return a->version() < b->version(); + }); +} + Status Compaction::do_compaction_impl() { OlapStopWatch watch; @@ -78,11 +84,14 @@ Status Compaction::do_compaction_impl() { << ", err=" << iterator_num_res.status().to_string(); return iterator_num_res.status(); } + + const TabletSchemaCSPtr& cur_tablet_schema = tablet_meta_with_max_rowset_version(_input_rowsets)->schema(); + size_t segment_iterator_num = iterator_num_res.value(); CompactionAlgorithm algorithm = CompactionUtils::choose_compaction_algorithm( _tablet->num_columns(), config::vertical_compaction_max_columns_per_group, segment_iterator_num); if (algorithm == VERTICAL_COMPACTION) { - CompactionUtils::split_column_into_groups(_tablet->num_columns(), _tablet->tablet_schema().sort_key_idxes(), + CompactionUtils::split_column_into_groups(_tablet->num_columns(), cur_tablet_schema->sort_key_idxes(), config::vertical_compaction_max_columns_per_group, &_column_groups); } @@ -96,16 +105,16 @@ Status Compaction::do_compaction_impl() { << ", column group size=" << _column_groups.size() << ", columns per group=" << config::vertical_compaction_max_columns_per_group; - RETURN_IF_ERROR(CompactionUtils::construct_output_rowset_writer(_tablet.get(), max_rows_per_segment, algorithm, - _output_version, &_output_rs_writer)); + RETURN_IF_ERROR(CompactionUtils::construct_output_rowset_writer( + _tablet.get(), max_rows_per_segment, algorithm, _output_version, &_output_rs_writer, &cur_tablet_schema)); TRACE("prepare finished"); Statistics stats; Status st; if (algorithm == VERTICAL_COMPACTION) { - st = _merge_rowsets_vertically(segment_iterator_num, &stats); + st = _merge_rowsets_vertically(segment_iterator_num, &stats, cur_tablet_schema); } else { - st = _merge_rowsets_horizontally(segment_iterator_num, &stats); + st = _merge_rowsets_horizontally(segment_iterator_num, &stats, cur_tablet_schema); } if (!st.ok()) { LOG(WARNING) << "fail to do " << compaction_name() << ". res=" << st << ", tablet=" << _tablet->tablet_id() @@ -152,10 +161,12 @@ Status Compaction::do_compaction_impl() { return Status::OK(); } -Status Compaction::_merge_rowsets_horizontally(size_t segment_iterator_num, Statistics* stats_output) { +Status Compaction::_merge_rowsets_horizontally(size_t segment_iterator_num, Statistics* stats_output, + const TabletSchemaCSPtr& tablet_schema) { TRACE_COUNTER_SCOPE_LATENCY_US("merge_rowsets_latency_us"); - Schema schema = ChunkHelper::convert_schema(_tablet->tablet_schema()); - TabletReader reader(_tablet, _output_rs_writer->version(), schema); + Schema schema = ChunkHelper::convert_schema(tablet_schema); + auto merge_tablet_schema = std::shared_ptr(TabletSchema::copy(tablet_schema)); + TabletReader reader(_tablet, _output_rs_writer->version(), merge_tablet_schema, schema); TabletReaderParams reader_params; reader_params.reader_type = compaction_type(); reader_params.profile = _runtime_profile.create_child("merge_rowsets"); @@ -200,7 +211,7 @@ Status Compaction::_merge_rowsets_horizontally(size_t segment_iterator_num, Stat } } - ChunkHelper::padding_char_columns(char_field_indexes, schema, _tablet->tablet_schema(), chunk.get()); + ChunkHelper::padding_char_columns(char_field_indexes, schema, tablet_schema, chunk.get()); if (auto st = _output_rs_writer->add_chunk(*chunk); !st.ok()) { LOG(WARNING) << "writer add_chunk error: " << st; @@ -227,7 +238,8 @@ Status Compaction::_merge_rowsets_horizontally(size_t segment_iterator_num, Stat return Status::OK(); } -Status Compaction::_merge_rowsets_vertically(size_t segment_iterator_num, Statistics* stats_output) { +Status Compaction::_merge_rowsets_vertically(size_t segment_iterator_num, Statistics* stats_output, + const TabletSchemaCSPtr& tablet_schema) { TRACE_COUNTER_SCOPE_LATENCY_US("merge_rowsets_latency_us"); auto mask_buffer = std::make_unique(_tablet->tablet_id(), _tablet->data_dir()->path()); auto source_masks = std::make_unique>(); @@ -238,7 +250,7 @@ Status Compaction::_merge_rowsets_vertically(size_t segment_iterator_num, Statis mask_buffer->flip_to_read(); } - Schema schema = ChunkHelper::convert_schema(_tablet->tablet_schema(), _column_groups[i]); + Schema schema = ChunkHelper::convert_schema(tablet_schema, _column_groups[i]); TabletReader reader(_tablet, _output_rs_writer->version(), schema, is_key, mask_buffer.get()); RETURN_IF_ERROR(reader.prepare()); TabletReaderParams reader_params; @@ -251,6 +263,9 @@ Status Compaction::_merge_rowsets_vertically(size_t segment_iterator_num, Statis total_num_rows += rowset->num_rows(); for (auto& segment : rowset->segments()) { for (uint32_t column_index : _column_groups[i]) { + if (!segment->is_valid_column(tablet_schema->column(column_index).unique_id())) { + continue; + } const auto* column_reader = segment->column(column_index); if (column_reader == nullptr) { continue; @@ -292,7 +307,7 @@ Status Compaction::_merge_rowsets_vertically(size_t segment_iterator_num, Statis } } - ChunkHelper::padding_char_columns(char_field_indexes, schema, _tablet->tablet_schema(), chunk.get()); + ChunkHelper::padding_char_columns(char_field_indexes, schema, tablet_schema, chunk.get()); if (auto st = _output_rs_writer->add_columns(*chunk, _column_groups[i], is_key); !st.ok()) { LOG(WARNING) << "writer add chunk by columns error. tablet=" << _tablet->tablet_id() << ", err=" << st; diff --git a/be/src/storage/compaction.h b/be/src/storage/compaction.h index c2e10a835790a..658739011a27d 100644 --- a/be/src/storage/compaction.h +++ b/be/src/storage/compaction.h @@ -91,8 +91,10 @@ class Compaction { // merge rows from vectorized reader and write into `_output_rs_writer`. // return Status::OK() and set statistics into `*stats_output`. // return others on error - Status _merge_rowsets_horizontally(size_t segment_iterator_num, Statistics* stats_output); - Status _merge_rowsets_vertically(size_t segment_iterator_num, Statistics* stats_output); + Status _merge_rowsets_horizontally(size_t segment_iterator_num, Statistics* stats_output, + const TabletSchemaCSPtr& tablet_schema); + Status _merge_rowsets_vertically(size_t segment_iterator_num, Statistics* stats_output, + const TabletSchemaCSPtr& tablet_schema); }; } // namespace starrocks diff --git a/be/src/storage/compaction_utils.cpp b/be/src/storage/compaction_utils.cpp index e27637da8f667..a21c3176794c5 100644 --- a/be/src/storage/compaction_utils.cpp +++ b/be/src/storage/compaction_utils.cpp @@ -52,7 +52,8 @@ int32_t CompactionUtils::get_read_chunk_size(int64_t mem_limit, int32_t config_c Status CompactionUtils::construct_output_rowset_writer(Tablet* tablet, uint32_t max_rows_per_segment, CompactionAlgorithm algorithm, Version version, - std::unique_ptr* output_rowset_writer) { + std::unique_ptr* output_rowset_writer, + const TabletSchemaCSPtr* tablet_schema) { RowsetWriterContext context; context.rowset_id = StorageEngine::instance()->next_rowset_id(); context.tablet_uid = tablet->tablet_uid(); @@ -60,7 +61,7 @@ Status CompactionUtils::construct_output_rowset_writer(Tablet* tablet, uint32_t context.partition_id = tablet->partition_id(); context.tablet_schema_hash = tablet->schema_hash(); context.rowset_path_prefix = tablet->schema_hash_path(); - context.tablet_schema = &tablet->tablet_schema(); + context.tablet_schema = !tablet_schema ? tablet->tablet_schema() : *tablet_schema; context.rowset_state = VISIBLE; context.version = version; context.segments_overlap = NONOVERLAPPING; diff --git a/be/src/storage/compaction_utils.h b/be/src/storage/compaction_utils.h index 2aaa0314223ea..fa7620c7aca9b 100644 --- a/be/src/storage/compaction_utils.h +++ b/be/src/storage/compaction_utils.h @@ -20,6 +20,7 @@ #include "common/status.h" #include "storage/olap_common.h" +#include "tablet_schema.h" namespace starrocks { @@ -51,7 +52,8 @@ class CompactionUtils { static Status construct_output_rowset_writer(Tablet* tablet, uint32_t max_rows_per_segment, CompactionAlgorithm algorithm, Version version, - std::unique_ptr* output_rowset_writer); + std::unique_ptr* output_rowset_writer, + const TabletSchemaCSPtr* tablet_schema = nullptr); static uint32_t get_segment_max_rows(int64_t max_segment_file_size, int64_t input_row_num, int64_t input_size); diff --git a/be/src/storage/data_dir.cpp b/be/src/storage/data_dir.cpp index 3cc3fd7317d17..d4ccb6bf172fb 100644 --- a/be/src/storage/data_dir.cpp +++ b/be/src/storage/data_dir.cpp @@ -346,6 +346,15 @@ Status DataDir::load() { << ", path: " << _path; } + for (int64_t tablet_id : tablet_ids) { + TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id); + if (tablet && tablet->set_tablet_schema_into_rowset_meta()) { + TabletMetaPB tablet_meta_pb; + tablet->tablet_meta()->to_meta_pb(&tablet_meta_pb); + TabletMetaManager::save(this, tablet_meta_pb); + } + } + // traverse rowset // 1. add committed rowset to txn map // 2. add visible rowset to tablet @@ -360,8 +369,8 @@ Status DataDir::load() { continue; } RowsetSharedPtr rowset; - Status create_status = RowsetFactory::create_rowset(&tablet->tablet_schema(), tablet->schema_hash_path(), - rowset_meta, &rowset); + Status create_status = + RowsetFactory::create_rowset(tablet->tablet_schema(), tablet->schema_hash_path(), rowset_meta, &rowset); if (!create_status.ok()) { LOG(WARNING) << "Fail to create rowset from rowsetmeta," << " rowset=" << rowset_meta->rowset_id() << " state=" << rowset_meta->rowset_state(); @@ -369,6 +378,11 @@ Status DataDir::load() { } if (rowset_meta->rowset_state() == RowsetStatePB::COMMITTED && rowset_meta->tablet_uid() == tablet->tablet_uid()) { + if (!rowset_meta->tablet_schema()) { + auto tablet_schema_ptr = tablet->tablet_schema(); + rowset_meta->set_tablet_schema(tablet_schema_ptr); + RowsetMetaManager::save(get_meta(), rowset_meta->tablet_uid(), rowset_meta->get_meta_pb()); + } Status commit_txn_status = _txn_manager->commit_txn( _kv_store, rowset_meta->partition_id(), rowset_meta->txn_id(), rowset_meta->tablet_id(), rowset_meta->tablet_schema_hash(), rowset_meta->tablet_uid(), rowset_meta->load_id(), rowset, true); @@ -381,9 +395,14 @@ Status DataDir::load() { << " schema hash=" << rowset_meta->tablet_schema_hash() << " txn_id: " << rowset_meta->txn_id(); } + } else if (rowset_meta->rowset_state() == RowsetStatePB::VISIBLE && rowset_meta->tablet_uid() == tablet->tablet_uid()) { Status publish_status = tablet->load_rowset(rowset); + if (!rowset_meta->tablet_schema()) { + rowset_meta->set_tablet_schema(tablet->tablet_schema()); + RowsetMetaManager::save(get_meta(), rowset_meta->tablet_uid(), rowset_meta->get_meta_pb()); + } if (!publish_status.ok() && !publish_status.is_already_exist()) { LOG(WARNING) << "Fail to add visible rowset=" << rowset->rowset_id() << " to tablet=" << rowset_meta->tablet_id() << " txn id=" << rowset_meta->txn_id() diff --git a/be/src/storage/delta_writer.cpp b/be/src/storage/delta_writer.cpp index a8bdef8dcec24..03b8587591d62 100644 --- a/be/src/storage/delta_writer.cpp +++ b/be/src/storage/delta_writer.cpp @@ -51,7 +51,7 @@ DeltaWriter::DeltaWriter(DeltaWriterOptions opt, MemTracker* mem_tracker, Storag _schema_initialized(false), _mem_table(nullptr), _mem_table_sink(nullptr), - _tablet_schema(nullptr), + _tablet_schema(new TabletSchema), _flush_token(nullptr), _replicate_token(nullptr), _with_rollback_log(true) {} @@ -132,6 +132,7 @@ Status DeltaWriter::_init() { return st; } } + if (_tablet->version_count() > config::tablet_max_versions) { if (config::enable_event_based_compaction_framework) { StorageEngine::instance()->compaction_manager()->update_tablet_async(_tablet); @@ -188,9 +189,14 @@ Status DeltaWriter::_init() { return _opt.slots->size(); } }(); + + // build tablet schema in request level + auto tablet_schema_ptr = _tablet->tablet_schema(); + _build_current_tablet_schema(_opt.index_id, _opt.ptable_schema_param, _tablet->tablet_schema()); + // maybe partial update, change to partial tablet schema - if (_tablet->tablet_schema().keys_type() == KeysType::PRIMARY_KEYS && - partial_cols_num < _tablet->tablet_schema().num_columns()) { + if (tablet_schema_ptr->keys_type() == KeysType::PRIMARY_KEYS && + partial_cols_num < tablet_schema_ptr->num_columns()) { writer_context.referenced_column_ids.reserve(partial_cols_num); for (auto i = 0; i < partial_cols_num; ++i) { const auto& slot_col_name = (*_opt.slots)[i]->col_name(); @@ -211,32 +217,32 @@ Status DeltaWriter::_init() { // If tablet is a new created tablet and has no historical data, average_row_size is 0 // And we use schema size as average row size. If there are complex type(i.e. BITMAP/ARRAY) or varchar, // we will consider it as 16 bytes. - average_row_size = _tablet->tablet_schema().estimate_row_size(16); + average_row_size = tablet_schema_ptr->estimate_row_size(16); _memtable_buffer_row = config::write_buffer_size / average_row_size; } - writer_context.partial_update_tablet_schema = - TabletSchema::create(_tablet->tablet_schema(), writer_context.referenced_column_ids); - auto sort_key_idxes = _tablet->tablet_schema().sort_key_idxes(); + TabletSchema::create(tablet_schema_ptr, writer_context.referenced_column_ids); + auto sort_key_idxes = tablet_schema_ptr->sort_key_idxes(); std::sort(sort_key_idxes.begin(), sort_key_idxes.end()); if (!std::includes(writer_context.referenced_column_ids.begin(), writer_context.referenced_column_ids.end(), sort_key_idxes.begin(), sort_key_idxes.end())) { _partial_schema_with_sort_key = true; } - writer_context.tablet_schema = writer_context.partial_update_tablet_schema.get(); + writer_context.tablet_schema = writer_context.partial_update_tablet_schema; + _tablet_schema = writer_context.partial_update_tablet_schema; writer_context.partial_update_mode = _opt.partial_update_mode; } else { - writer_context.tablet_schema = &_tablet->tablet_schema(); - if (_tablet->tablet_schema().keys_type() == KeysType::PRIMARY_KEYS && !_opt.merge_condition.empty()) { + if (tablet_schema_ptr->keys_type() == KeysType::PRIMARY_KEYS && !_opt.merge_condition.empty()) { writer_context.merge_condition = _opt.merge_condition; } + writer_context.tablet_schema = _tablet_schema; } - auto sort_key_idxes = _tablet->tablet_schema().sort_key_idxes(); + auto sort_key_idxes = _tablet->tablet_schema()->sort_key_idxes(); std::sort(sort_key_idxes.begin(), sort_key_idxes.end()); bool auto_increment_in_sort_key = false; for (auto& idx : sort_key_idxes) { - auto& col = _tablet->tablet_schema().column(idx); + auto& col = _tablet->tablet_schema()->column(idx); if (col.is_auto_increment()) { auto_increment_in_sort_key = true; break; @@ -247,7 +253,6 @@ Status DeltaWriter::_init() { LOG(WARNING) << "auto increment column in sort key do not support partial update"; return Status::NotSupported("auto increment column in sort key do not support partial update"); } - writer_context.rowset_id = _storage_engine->next_rowset_id(); writer_context.tablet_uid = _tablet->tablet_uid(); writer_context.tablet_id = _opt.tablet_id; @@ -270,7 +275,6 @@ Status DeltaWriter::_init() { return st; } _mem_table_sink = std::make_unique(_rowset_writer.get()); - _tablet_schema = writer_context.tablet_schema; _flush_token = _storage_engine->memtable_flush_executor()->create_flush_token(); if (_replica_state == Primary && _opt.replicas.size() > 1) { _replicate_token = _storage_engine->segment_replicate_executor()->create_replicate_token(&_opt); @@ -447,12 +451,32 @@ Status DeltaWriter::_flush_memtable() { return _flush_token->wait(); } +void DeltaWriter::_build_current_tablet_schema(int64_t index_id, const POlapTableSchemaParam& ptable_schema_param, + const TabletSchemaCSPtr& ori_tablet_schema) { + _tablet_schema->copy_from(ori_tablet_schema); + // new tablet schema if new table + + // find the right index id + int i = 0; + for (; i < ptable_schema_param.indexes_size(); i++) { + if (ptable_schema_param.indexes(i).id() == index_id) break; + } + if (ptable_schema_param.indexes_size() > 0 && ptable_schema_param.indexes(0).columns_desc_size() != 0 && + ptable_schema_param.indexes(0).columns_desc(0).unique_id() >= 0) { + _tablet_schema->build_current_tablet_schema(index_id, ptable_schema_param.version(), + ptable_schema_param.indexes(i), ori_tablet_schema); + } + if (_tablet_schema->schema_version() > ori_tablet_schema->schema_version()) { + _tablet->update_max_version_schema(_tablet_schema); + } +} + void DeltaWriter::_reset_mem_table() { if (!_schema_initialized) { _vectorized_schema = MemTable::convert_schema(_tablet_schema, _opt.slots); _schema_initialized = true; } - if (_tablet->tablet_schema().keys_type() == KeysType::PRIMARY_KEYS && !_opt.merge_condition.empty()) { + if (_tablet->tablet_schema()->keys_type() == KeysType::PRIMARY_KEYS && !_opt.merge_condition.empty()) { _mem_table = std::make_unique(_tablet->tablet_id(), &_vectorized_schema, _opt.slots, _mem_table_sink.get(), _opt.merge_condition, _mem_tracker); } else { @@ -507,7 +531,7 @@ Status DeltaWriter::commit() { return res.status(); } - _cur_rowset->set_schema(&_tablet->tablet_schema()); + _cur_rowset->set_schema(_tablet->tablet_schema()); if (_tablet->keys_type() == KeysType::PRIMARY_KEYS) { auto st = _storage_engine->update_manager()->on_rowset_finished(_tablet.get(), _cur_rowset.get()); if (!st.ok()) { @@ -612,7 +636,7 @@ Status DeltaWriter::_fill_auto_increment_id(const Chunk& chunk) { for (size_t i = 0; i < _tablet_schema->num_key_columns(); i++) { pk_columns.push_back((uint32_t)i); } - Schema pkey_schema = ChunkHelper::convert_schema(*_tablet_schema, pk_columns); + Schema pkey_schema = ChunkHelper::convert_schema(_tablet_schema, pk_columns); std::unique_ptr pk_column; if (!PrimaryKeyEncoder::create_column(pkey_schema, &pk_column).ok()) { CHECK(false) << "create column for primary key encoder failed"; diff --git a/be/src/storage/delta_writer.h b/be/src/storage/delta_writer.h index 77ae7730de31e..2f6cf982b8ff4 100644 --- a/be/src/storage/delta_writer.h +++ b/be/src/storage/delta_writer.h @@ -65,6 +65,7 @@ struct DeltaWriterOptions { ReplicaState replica_state; bool miss_auto_increment_column = false; PartialUpdateMode partial_update_mode = PartialUpdateMode::UNKNOWN_MODE; + POlapTableSchemaParam ptable_schema_param; }; enum State { @@ -154,6 +155,9 @@ class DeltaWriter { Status _init(); Status _flush_memtable_async(bool eos = false); Status _flush_memtable(); + void _build_current_tablet_schema(int64_t index_id, const POlapTableSchemaParam& table_schema_param, + const TabletSchemaCSPtr& ori_tablet_schema); + const char* _state_name(State state) const; const char* _replica_state_name(ReplicaState state) const; Status _fill_auto_increment_id(const Chunk& chunk); @@ -181,7 +185,10 @@ class DeltaWriter { Schema _vectorized_schema; std::unique_ptr _mem_table; std::unique_ptr _mem_table_sink; - const TabletSchema* _tablet_schema; + // tablet schema owned by delta writer, all write will use this tablet schema + // it's build from unsafe_tablet_schema_ref（stored when create tablet） and OlapTableSchema + // every request will have it's own tablet schema so simple schema change can work + TabletSchemaSPtr _tablet_schema; std::unique_ptr _flush_token; std::unique_ptr _replicate_token; diff --git a/be/src/storage/horizontal_compaction_task.cpp b/be/src/storage/horizontal_compaction_task.cpp index fda57c4d66bd7..0713544350254 100644 --- a/be/src/storage/horizontal_compaction_task.cpp +++ b/be/src/storage/horizontal_compaction_task.cpp @@ -150,7 +150,8 @@ StatusOr HorizontalCompactionTask::_compact_data(int32_t chunk_size, Tab } } - ChunkHelper::padding_char_columns(char_field_indexes, schema, _tablet->tablet_schema(), chunk.get()); + auto tablet_schema_ptr = _tablet->tablet_schema(); + ChunkHelper::padding_char_columns(char_field_indexes, schema, tablet_schema_ptr, chunk.get()); RETURN_IF_ERROR(output_rs_writer->add_chunk(*chunk)); output_rows += chunk->num_rows(); diff --git a/be/src/storage/lake/delta_writer.cpp b/be/src/storage/lake/delta_writer.cpp index ad1f90f1cc730..fb959f8a27af0 100644 --- a/be/src/storage/lake/delta_writer.cpp +++ b/be/src/storage/lake/delta_writer.cpp @@ -230,7 +230,7 @@ Status DeltaWriterImpl::build_schema_and_writer() { inline Status DeltaWriterImpl::reset_memtable() { RETURN_IF_ERROR(build_schema_and_writer()); if (!_schema_initialized) { - _vectorized_schema = MemTable::convert_schema(_tablet_schema.get(), _slots); + _vectorized_schema = MemTable::convert_schema(_tablet_schema, _slots); _schema_initialized = true; } if (_slots != nullptr || !_merge_condition.empty()) { @@ -312,7 +312,7 @@ Status DeltaWriterImpl::handle_partial_update() { } _referenced_column_ids.push_back(index); } - _partial_update_tablet_schema = TabletSchema::create(*_tablet_schema, _referenced_column_ids); + _partial_update_tablet_schema = TabletSchema::create(_tablet_schema, _referenced_column_ids); auto sort_key_idxes = _tablet_schema->sort_key_idxes(); std::sort(sort_key_idxes.begin(), sort_key_idxes.end()); if (!std::includes(_referenced_column_ids.begin(), _referenced_column_ids.end(), sort_key_idxes.begin(), @@ -433,7 +433,7 @@ Status DeltaWriterImpl::_fill_auto_increment_id(const Chunk& chunk) { for (size_t i = 0; i < _tablet_schema->num_key_columns(); i++) { pk_columns.push_back((uint32_t)i); } - Schema pkey_schema = ChunkHelper::convert_schema(*_tablet_schema, pk_columns); + Schema pkey_schema = ChunkHelper::convert_schema(_tablet_schema, pk_columns); std::unique_ptr pk_column; if (!PrimaryKeyEncoder::create_column(pkey_schema, &pk_column).ok()) { CHECK(false) << "create column for primary key encoder failed"; diff --git a/be/src/storage/lake/general_tablet_writer.cpp b/be/src/storage/lake/general_tablet_writer.cpp index 59ff2fc8c58c5..2defc10e34c10 100644 --- a/be/src/storage/lake/general_tablet_writer.cpp +++ b/be/src/storage/lake/general_tablet_writer.cpp @@ -78,7 +78,7 @@ Status HorizontalGeneralTabletWriter::reset_segment_writer() { auto name = gen_segment_filename(_txn_id); ASSIGN_OR_RETURN(auto of, fs::new_writable_file(_tablet.segment_location(name))); SegmentWriterOptions opts; - auto w = std::make_unique(std::move(of), _seg_id++, _schema.get(), opts); + auto w = std::make_unique(std::move(of), _seg_id++, _schema, opts); RETURN_IF_ERROR(w->init()); _seg_writer = std::move(w); _files.emplace_back(std::move(name)); @@ -229,7 +229,7 @@ StatusOr> VerticalGeneralTabletWriter::create_seg auto name = gen_segment_filename(_txn_id); ASSIGN_OR_RETURN(auto of, fs::new_writable_file(_tablet.segment_location(name))); SegmentWriterOptions opts; - auto w = std::make_unique(std::move(of), _seg_id++, _schema.get(), opts); + auto w = std::make_unique(std::move(of), _seg_id++, _schema, opts); RETURN_IF_ERROR(w->init(column_indexes, is_key)); _files.emplace_back(std::move(name)); return w; diff --git a/be/src/storage/lake/horizontal_compaction_task.cpp b/be/src/storage/lake/horizontal_compaction_task.cpp index 99d32eb404ca8..0456fe788ea20 100644 --- a/be/src/storage/lake/horizontal_compaction_task.cpp +++ b/be/src/storage/lake/horizontal_compaction_task.cpp @@ -45,7 +45,7 @@ Status HorizontalCompactionTask::execute(Progress* progress, CancelFunc cancel_f VLOG(3) << "Start horizontal compaction. tablet: " << _tablet->id() << ", reader chunk size: " << chunk_size; - Schema schema = ChunkHelper::convert_schema(*tablet_schema); + Schema schema = ChunkHelper::convert_schema(tablet_schema); TabletReader reader(*_tablet, _version, schema, _input_rowsets); RETURN_IF_ERROR(reader.prepare()); TabletReaderParams reader_params; @@ -78,7 +78,7 @@ Status HorizontalCompactionTask::execute(Progress* progress, CancelFunc cancel_f } else if (!st.ok()) { return st; } - ChunkHelper::padding_char_columns(char_field_indexes, schema, *tablet_schema, chunk.get()); + ChunkHelper::padding_char_columns(char_field_indexes, schema, tablet_schema, chunk.get()); RETURN_IF_ERROR(writer->write(*chunk)); chunk->reset(); diff --git a/be/src/storage/lake/lake_local_persistent_index.cpp b/be/src/storage/lake/lake_local_persistent_index.cpp index 94a0c7c7baf29..6c7f8a4730e8f 100644 --- a/be/src/storage/lake/lake_local_persistent_index.cpp +++ b/be/src/storage/lake/lake_local_persistent_index.cpp @@ -112,12 +112,12 @@ Status LakeLocalPersistentIndex::load_from_lake_tablet(starrocks::lake::Tablet* } // 1. create and set key column schema - std::unique_ptr tablet_schema = std::make_unique(metadata.schema()); + std::shared_ptr tablet_schema = std::make_unique(metadata.schema()); vector pk_columns(tablet_schema->num_key_columns()); for (auto i = 0; i < tablet_schema->num_key_columns(); i++) { pk_columns[i] = (ColumnId)i; } - auto pkey_schema = ChunkHelper::convert_schema(*tablet_schema, pk_columns); + auto pkey_schema = ChunkHelper::convert_schema(tablet_schema, pk_columns); size_t fix_size = PrimaryKeyEncoder::get_encoded_fixed_size(pkey_schema); diff --git a/be/src/storage/lake/lake_primary_index.cpp b/be/src/storage/lake/lake_primary_index.cpp index 7c95b8451dc65..e547c2714d335 100644 --- a/be/src/storage/lake/lake_primary_index.cpp +++ b/be/src/storage/lake/lake_primary_index.cpp @@ -51,12 +51,12 @@ Status LakePrimaryIndex::_do_lake_load(Tablet* tablet, const TabletMetadata& met MonotonicStopWatch watch; watch.start(); // 1. create and set key column schema - std::unique_ptr tablet_schema = std::make_unique(metadata.schema()); + std::shared_ptr tablet_schema = std::make_shared(metadata.schema()); vector pk_columns(tablet_schema->num_key_columns()); for (auto i = 0; i < tablet_schema->num_key_columns(); i++) { pk_columns[i] = (ColumnId)i; } - auto pkey_schema = ChunkHelper::convert_schema(*tablet_schema, pk_columns); + auto pkey_schema = ChunkHelper::convert_schema(tablet_schema, pk_columns); _set_schema(pkey_schema); // load persistent index if enable persistent index meta diff --git a/be/src/storage/lake/rowset_update_state.cpp b/be/src/storage/lake/rowset_update_state.cpp index 96be47f3f579d..45ce6ca38130b 100644 --- a/be/src/storage/lake/rowset_update_state.cpp +++ b/be/src/storage/lake/rowset_update_state.cpp @@ -67,20 +67,20 @@ Status RowsetUpdateState::load(const TxnLogPB_OpWrite& op_write, const TabletMet } Status RowsetUpdateState::_do_load(const TxnLogPB_OpWrite& op_write, const TabletMetadata& metadata, Tablet* tablet) { - std::unique_ptr tablet_schema = std::make_unique(metadata.schema()); + std::shared_ptr tablet_schema = std::make_shared(metadata.schema()); std::unique_ptr rowset_ptr = std::make_unique(tablet, std::make_shared(op_write.rowset())); - RETURN_IF_ERROR(_do_load_upserts_deletes(op_write, *tablet_schema, tablet, rowset_ptr.get())); + RETURN_IF_ERROR(_do_load_upserts_deletes(op_write, tablet_schema, tablet, rowset_ptr.get())); if (!op_write.has_txn_meta() || rowset_ptr->num_segments() == 0 || op_write.txn_meta().has_merge_condition()) { return Status::OK(); } if (!op_write.txn_meta().partial_update_column_ids().empty()) { - RETURN_IF_ERROR(_prepare_partial_update_states(op_write, metadata, tablet, *tablet_schema)); + RETURN_IF_ERROR(_prepare_partial_update_states(op_write, metadata, tablet, tablet_schema)); } if (op_write.txn_meta().has_auto_increment_partial_update_column_id()) { - RETURN_IF_ERROR(_prepare_auto_increment_partial_update_states(op_write, metadata, tablet, *tablet_schema)); + RETURN_IF_ERROR(_prepare_auto_increment_partial_update_states(op_write, metadata, tablet, tablet_schema)); } return Status::OK(); } @@ -160,10 +160,11 @@ void RowsetUpdateState::plan_read_by_rssid(const std::vector& rowids, } } -Status RowsetUpdateState::_do_load_upserts_deletes(const TxnLogPB_OpWrite& op_write, const TabletSchema& tablet_schema, - Tablet* tablet, Rowset* rowset_ptr) { +Status RowsetUpdateState::_do_load_upserts_deletes(const TxnLogPB_OpWrite& op_write, + const TabletSchemaCSPtr& tablet_schema, Tablet* tablet, + Rowset* rowset_ptr) { vector pk_columns; - for (size_t i = 0; i < tablet_schema.num_key_columns(); i++) { + for (size_t i = 0; i < tablet_schema->num_key_columns(); i++) { pk_columns.push_back((uint32_t)i); } Schema pkey_schema = ChunkHelper::convert_schema(tablet_schema, pk_columns); @@ -240,7 +241,8 @@ Status RowsetUpdateState::_do_load_upserts_deletes(const TxnLogPB_OpWrite& op_wr return Status::OK(); } -static std::vector get_read_columns_ids(const TxnLogPB_OpWrite& op_write, const TabletSchema& tablet_schema) { +static std::vector get_read_columns_ids(const TxnLogPB_OpWrite& op_write, + const TabletSchemaCSPtr& tablet_schema) { const auto& txn_meta = op_write.txn_meta(); std::vector update_column_ids(txn_meta.partial_update_column_ids().begin(), @@ -248,7 +250,7 @@ static std::vector get_read_columns_ids(const TxnLogPB_OpWrite& op_wri std::set update_columns_set(update_column_ids.begin(), update_column_ids.end()); std::vector read_column_ids; - for (uint32_t i = 0; i < tablet_schema.num_columns(); i++) { + for (uint32_t i = 0; i < tablet_schema->num_columns(); i++) { if (update_columns_set.find(i) == update_columns_set.end()) { read_column_ids.push_back(i); } @@ -259,15 +261,15 @@ static std::vector get_read_columns_ids(const TxnLogPB_OpWrite& op_wri Status RowsetUpdateState::_prepare_auto_increment_partial_update_states(const TxnLogPB_OpWrite& op_write, const TabletMetadata& metadata, Tablet* tablet, - const TabletSchema& tablet_schema) { + const TabletSchemaCSPtr& tablet_schema) { const auto& txn_meta = op_write.txn_meta(); size_t num_segments = op_write.rowset().segments_size(); _auto_increment_partial_update_states.resize(num_segments); _auto_increment_delete_pks.resize(num_segments); uint32_t auto_increment_column_id = 0; - for (int i = 0; i < tablet_schema.num_columns(); ++i) { - if (tablet_schema.column(i).is_auto_increment()) { + for (int i = 0; i < tablet_schema->num_columns(); ++i) { + if (tablet_schema->column(i).is_auto_increment()) { auto_increment_column_id = i; break; } @@ -284,7 +286,7 @@ Status RowsetUpdateState::_prepare_auto_increment_partial_update_states(const Tx schema = TabletSchema::create(tablet_schema, update_column_ids); } else { std::vector all_column_ids; - all_column_ids.resize(tablet_schema.num_columns()); + all_column_ids.resize(tablet_schema->num_columns()); std::iota(all_column_ids.begin(), all_column_ids.end(), 0); schema = TabletSchema::create(tablet_schema, all_column_ids); } @@ -383,7 +385,7 @@ Status RowsetUpdateState::_prepare_auto_increment_partial_update_states(const Tx Status RowsetUpdateState::_prepare_partial_update_states(const TxnLogPB_OpWrite& op_write, const TabletMetadata& metadata, Tablet* tablet, - const TabletSchema& tablet_schema) { + const TabletSchemaCSPtr& tablet_schema) { int64_t t_start = MonotonicMillis(); std::vector read_column_ids = get_read_columns_ids(op_write, tablet_schema); @@ -453,7 +455,7 @@ Status RowsetUpdateState::rewrite_segment(const TxnLogPB_OpWrite& op_write, cons RowsetMetadata* rowset_meta = const_cast(&op_write)->mutable_rowset(); auto root_path = tablet->metadata_root_location(); ASSIGN_OR_RETURN(auto fs, FileSystem::CreateSharedFromString(root_path)); - std::unique_ptr tablet_schema = std::make_unique(metadata.schema()); + std::shared_ptr tablet_schema = std::make_shared(metadata.schema()); // get rowset schema if (!op_write.has_txn_meta() || op_write.rewrite_segments_size() == 0 || rowset_meta->segments_size() == 0 || op_write.txn_meta().has_merge_condition()) { @@ -486,7 +488,7 @@ Status RowsetUpdateState::rewrite_segment(const TxnLogPB_OpWrite& op_write, cons if (op_write.txn_meta().has_auto_increment_partial_update_column_id() && !_auto_increment_partial_update_states[i].skip_rewrite) { RETURN_IF_ERROR(SegmentRewriter::rewrite( - tablet->segment_location(src_path), tablet->segment_location(dest_path), *tablet_schema, + tablet->segment_location(src_path), tablet->segment_location(dest_path), tablet_schema, _auto_increment_partial_update_states[i], read_column_ids, _partial_update_states.size() != 0 ? &_partial_update_states[i].write_columns : nullptr, op_write, tablet)); @@ -494,7 +496,7 @@ Status RowsetUpdateState::rewrite_segment(const TxnLogPB_OpWrite& op_write, cons const FooterPointerPB& partial_rowset_footer = txn_meta.partial_rowset_footers(i); // if rewrite fail, let segment gc to clean dest segment file RETURN_IF_ERROR(SegmentRewriter::rewrite( - tablet->segment_location(src_path), tablet->segment_location(dest_path), *tablet_schema, + tablet->segment_location(src_path), tablet->segment_location(dest_path), tablet_schema, read_column_ids, _partial_update_states[i].write_columns, i, partial_rowset_footer)); } else { need_rename[i] = false; @@ -546,8 +548,8 @@ Status RowsetUpdateState::_resolve_conflict(const TxnLogPB_OpWrite& op_write, co tablet->update_mgr()->get_rowids_from_pkindex(tablet, _base_version, _upserts, &new_rss_rowids_vec)); size_t total_conflicts = 0; - std::unique_ptr tablet_schema = std::make_unique(metadata.schema()); - std::vector read_column_ids = get_read_columns_ids(op_write, *tablet_schema); + std::shared_ptr tablet_schema = std::make_shared(metadata.schema()); + std::vector read_column_ids = get_read_columns_ids(op_write, tablet_schema); // get rss_rowids to identify conflict exist or not int64_t t_start = MonotonicMillis(); for (uint32_t segment_id = 0; segment_id < num_segments; segment_id++) { @@ -557,13 +559,13 @@ Status RowsetUpdateState::_resolve_conflict(const TxnLogPB_OpWrite& op_write, co if (!op_write.txn_meta().partial_update_column_ids().empty()) { RETURN_IF_ERROR(_resolve_conflict_partial_update(op_write, metadata, tablet, new_rss_rowids, read_column_ids, segment_id, total_conflicts, - tablet_schema.get())); + tablet_schema)); } // reslove auto increment if (op_write.txn_meta().has_auto_increment_partial_update_column_id()) { RETURN_IF_ERROR(_resolve_conflict_auto_increment(op_write, metadata, tablet, new_rss_rowids, segment_id, - total_conflicts, tablet_schema.get())); + total_conflicts, tablet_schema)); } } int64_t t_end = MonotonicMillis(); @@ -579,7 +581,8 @@ Status RowsetUpdateState::_resolve_conflict_partial_update(const TxnLogPB_OpWrit const TabletMetadata& metadata, Tablet* tablet, const std::vector& new_rss_rowids, std::vector& read_column_ids, uint32_t segment_id, - size_t& total_conflicts, TabletSchema* tablet_schema) { + size_t& total_conflicts, + const TabletSchemaCSPtr& tablet_schema) { uint32_t num_rows = new_rss_rowids.size(); std::vector conflict_idxes; std::vector conflict_rowids; @@ -607,7 +610,7 @@ Status RowsetUpdateState::_resolve_conflict_partial_update(const TxnLogPB_OpWrit std::vector read_idxes; plan_read_by_rssid(conflict_rowids, &num_default, &rowids_by_rssid, &read_idxes); DCHECK_EQ(conflict_idxes.size(), read_idxes.size()); - RETURN_IF_ERROR(tablet->update_mgr()->get_column_values(tablet, metadata, op_write, *tablet_schema, + RETURN_IF_ERROR(tablet->update_mgr()->get_column_values(tablet, metadata, op_write, tablet_schema, read_column_ids, num_default > 0, rowids_by_rssid, &read_columns)); @@ -627,7 +630,7 @@ Status RowsetUpdateState::_resolve_conflict_auto_increment(const TxnLogPB_OpWrit const TabletMetadata& metadata, Tablet* tablet, const std::vector& new_rss_rowids, uint32_t segment_id, size_t& total_conflicts, - TabletSchema* tablet_schema) { + const TabletSchemaCSPtr& tablet_schema) { uint32_t num_rows = new_rss_rowids.size(); std::vector conflict_idxes; std::vector conflict_rowids; @@ -689,7 +692,7 @@ Status RowsetUpdateState::_resolve_conflict_auto_increment(const TxnLogPB_OpWrit auto_increment_read_column.resize(1); auto_increment_read_column[0] = _auto_increment_partial_update_states[segment_id].write_column->clone_empty(); RETURN_IF_ERROR(tablet->update_mgr()->get_column_values( - tablet, metadata, op_write, *tablet_schema, column_id, new_rows > 0, rowids_by_rssid, + tablet, metadata, op_write, tablet_schema, column_id, new_rows > 0, rowids_by_rssid, &auto_increment_read_column, &_auto_increment_partial_update_states[segment_id])); std::unique_ptr new_write_column = diff --git a/be/src/storage/lake/rowset_update_state.h b/be/src/storage/lake/rowset_update_state.h index 0414914e62c29..1a10ef1715814 100644 --- a/be/src/storage/lake/rowset_update_state.h +++ b/be/src/storage/lake/rowset_update_state.h @@ -81,11 +81,11 @@ class RowsetUpdateState { private: Status _do_load(const TxnLogPB_OpWrite& op_write, const TabletMetadata& metadata, Tablet* tablet); - Status _do_load_upserts_deletes(const TxnLogPB_OpWrite& op_write, const TabletSchema& tablet_schema, Tablet* tablet, - Rowset* rowset_ptr); + Status _do_load_upserts_deletes(const TxnLogPB_OpWrite& op_write, const TabletSchemaCSPtr& tablet_schema, + Tablet* tablet, Rowset* rowset_ptr); Status _prepare_partial_update_states(const TxnLogPB_OpWrite& op_write, const TabletMetadata& metadata, - Tablet* tablet, const TabletSchema& tablet_schema); + Tablet* tablet, const TabletSchemaCSPtr& tablet_schema); Status _resolve_conflict(const TxnLogPB_OpWrite& op_write, const TabletMetadata& metadata, int64_t base_version, Tablet* tablet, const MetaFileBuilder* builder); @@ -93,15 +93,16 @@ class RowsetUpdateState { Status _resolve_conflict_partial_update(const TxnLogPB_OpWrite& op_write, const TabletMetadata& metadata, Tablet* tablet, const std::vector& new_rss_rowids, std::vector& read_column_ids, uint32_t segment_id, - size_t& total_conflicts, TabletSchema* tablet_schema); + size_t& total_conflicts, const TabletSchemaCSPtr& tablet_schema); Status _resolve_conflict_auto_increment(const TxnLogPB_OpWrite& op_write, const TabletMetadata& metadata, Tablet* tablet, const std::vector& new_rss_rowids, - uint32_t segment_id, size_t& total_conflicts, TabletSchema* tablet_schema); + uint32_t segment_id, size_t& total_conflicts, + const TabletSchemaCSPtr& tablet_schema); Status _prepare_auto_increment_partial_update_states(const TxnLogPB_OpWrite& op_write, const TabletMetadata& metadata, Tablet* tablet, - const TabletSchema& tablet_schema); + const TabletSchemaCSPtr& tablet_schema); std::once_flag _load_once_flag; Status _status; diff --git a/be/src/storage/lake/schema_change.cpp b/be/src/storage/lake/schema_change.cpp index ddb339033243f..fb2596e4f4636 100644 --- a/be/src/storage/lake/schema_change.cpp +++ b/be/src/storage/lake/schema_change.cpp @@ -137,9 +137,9 @@ Status ConvertedSchemaChange::init() { _read_params.fill_data_cache = false; ASSIGN_OR_RETURN(auto base_tablet_schema, _base_tablet->get_schema()); - _base_schema = ChunkHelper::convert_schema(*base_tablet_schema, _chunk_changer->get_selected_column_indexes()); + _base_schema = ChunkHelper::convert_schema(base_tablet_schema, _chunk_changer->get_selected_column_indexes()); ASSIGN_OR_RETURN(_new_tablet_schema, _new_tablet->get_schema()); - _new_schema = ChunkHelper::convert_schema(*_new_tablet_schema); + _new_schema = ChunkHelper::convert_schema(_new_tablet_schema); _base_chunk = ChunkHelper::new_chunk(_base_schema, config::vector_chunk_size); _new_chunk = ChunkHelper::new_chunk(_new_schema, config::vector_chunk_size); @@ -183,7 +183,7 @@ Status DirectSchemaChange::process(RowsetPtr rowset, RowsetMetadata* new_rowset_ return Status::InternalError("failed to convert chunk data"); } - ChunkHelper::padding_char_columns(_char_field_indexes, _new_schema, *_new_tablet_schema, _new_chunk.get()); + ChunkHelper::padding_char_columns(_char_field_indexes, _new_schema, _new_tablet_schema, _new_chunk.get()); RETURN_IF_ERROR(writer->write(*_new_chunk)); } @@ -259,7 +259,7 @@ Status SortedSchemaChange::process(RowsetPtr rowset, RowsetMetadata* new_rowset_ return Status::InternalError("failed to convert chunk data"); } - ChunkHelper::padding_char_columns(_char_field_indexes, _new_schema, *_new_tablet_schema, _new_chunk.get()); + ChunkHelper::padding_char_columns(_char_field_indexes, _new_schema, _new_tablet_schema, _new_chunk.get()); RETURN_IF_ERROR(writer->write(*_new_chunk, _selective->data(), _new_chunk->num_rows())); } @@ -304,12 +304,12 @@ Status SchemaChangeHandler::do_process_alter_tablet(const TAlterTabletReqV2& req SchemaChangeParams sc_params; sc_params.base_tablet = &base_tablet; sc_params.new_tablet = &new_tablet; - sc_params.chunk_changer = std::make_unique(*new_schema); + sc_params.chunk_changer = std::make_unique(new_schema); sc_params.version = alter_version; sc_params.txn_id = request.txn_id; SchemaChangeUtils::init_materialized_params(request, &sc_params.materialized_params_map); - RETURN_IF_ERROR(SchemaChangeUtils::parse_request(*base_schema, *new_schema, sc_params.chunk_changer.get(), + RETURN_IF_ERROR(SchemaChangeUtils::parse_request(base_schema, new_schema, sc_params.chunk_changer.get(), sc_params.materialized_params_map, has_delete_predicates, &sc_params.sc_sorting, &sc_params.sc_directly, nullptr)); diff --git a/be/src/storage/lake/spark_load.cpp b/be/src/storage/lake/spark_load.cpp index ed8566e5c4574..8c054bfa12d81 100644 --- a/be/src/storage/lake/spark_load.cpp +++ b/be/src/storage/lake/spark_load.cpp @@ -54,7 +54,7 @@ Status SparkLoadHandler::_load_convert(Tablet& cur_tablet) { DeferOp defer([&]() { writer->close(); }); ASSIGN_OR_RETURN(auto tablet_schema, cur_tablet.get_schema()); - Schema schema = ChunkHelper::convert_schema(*tablet_schema); + Schema schema = ChunkHelper::convert_schema(tablet_schema); ChunkPtr chunk = ChunkHelper::new_chunk(schema, 0); auto char_field_indexes = ChunkHelper::get_char_field_indexes(schema); @@ -96,7 +96,7 @@ Status SparkLoadHandler::_load_convert(Tablet& cur_tablet) { break; } - ChunkHelper::padding_char_columns(char_field_indexes, schema, *tablet_schema, chunk.get()); + ChunkHelper::padding_char_columns(char_field_indexes, schema, tablet_schema, chunk.get()); RETURN_IF_ERROR(writer->write(*chunk)); chunk->reset(); } diff --git a/be/src/storage/lake/tablet_reader.cpp b/be/src/storage/lake/tablet_reader.cpp index a4e856e955725..58470b1a05047 100644 --- a/be/src/storage/lake/tablet_reader.cpp +++ b/be/src/storage/lake/tablet_reader.cpp @@ -132,7 +132,7 @@ Status TabletReader::get_segment_iterators(const TabletReaderParams& params, std rs_opts.runtime_state = params.runtime_state; rs_opts.profile = params.profile; rs_opts.use_page_cache = params.use_page_cache; - rs_opts.tablet_schema = _tablet_schema.get(); + rs_opts.tablet_schema = _tablet_schema; rs_opts.global_dictmaps = params.global_dictmaps; rs_opts.unused_output_column_ids = params.unused_output_column_ids; rs_opts.runtime_range_pruner = params.runtime_range_pruner; @@ -158,7 +158,7 @@ Status TabletReader::init_predicates(const TabletReaderParams& params) { } Status TabletReader::init_delete_predicates(const TabletReaderParams& params, DeletePredicates* dels) { - PredicateParser pred_parser(*_tablet_schema); + PredicateParser pred_parser(_tablet_schema); ASSIGN_OR_RETURN(auto tablet_metadata, enhance_error_prompt(_tablet.get_metadata(_version))); for (int index = 0, size = tablet_metadata->rowsets_size(); index < size; ++index) { diff --git a/be/src/storage/lake/tablet_writer.h b/be/src/storage/lake/tablet_writer.h index 4114d925556d3..b0c992ca8507f 100644 --- a/be/src/storage/lake/tablet_writer.h +++ b/be/src/storage/lake/tablet_writer.h @@ -100,11 +100,11 @@ class TabletWriter { virtual RowsetTxnMetaPB* rowset_txn_meta() = 0; // allow to set custom tablet schema for writer, used in partial update - void set_tablet_schema(std::shared_ptr schema) { _schema = std::move(schema); } + void set_tablet_schema(TabletSchemaCSPtr schema) { _schema = std::move(schema); } protected: Tablet _tablet; - std::shared_ptr _schema; + TabletSchemaCSPtr _schema; int64_t _txn_id; std::vector _files; int64_t _num_rows = 0; diff --git a/be/src/storage/lake/update_compaction_state.cpp b/be/src/storage/lake/update_compaction_state.cpp index ba10cb7852cf2..62a99b1e696ba 100644 --- a/be/src/storage/lake/update_compaction_state.cpp +++ b/be/src/storage/lake/update_compaction_state.cpp @@ -31,7 +31,7 @@ CompactionState::CompactionState(Rowset* rowset) { CompactionState::~CompactionState() = default; -Status CompactionState::load_segments(Rowset* rowset, const TabletSchema& tablet_schema, uint32_t segment_id) { +Status CompactionState::load_segments(Rowset* rowset, const TabletSchemaCSPtr& tablet_schema, uint32_t segment_id) { if (segment_id >= pk_cols.size() && pk_cols.size() != 0) { std::string msg = strings::Substitute("Error segment id: $0 vs $1", segment_id, pk_cols.size()); LOG(WARNING) << msg; @@ -43,9 +43,9 @@ Status CompactionState::load_segments(Rowset* rowset, const TabletSchema& tablet return _load_segments(rowset, tablet_schema, segment_id); } -Status CompactionState::_load_segments(Rowset* rowset, const TabletSchema& tablet_schema, uint32_t segment_id) { +Status CompactionState::_load_segments(Rowset* rowset, const TabletSchemaCSPtr& tablet_schema, uint32_t segment_id) { vector pk_columns; - for (size_t i = 0; i < tablet_schema.num_key_columns(); i++) { + for (size_t i = 0; i < tablet_schema->num_key_columns(); i++) { pk_columns.push_back(static_cast(i)); } diff --git a/be/src/storage/lake/update_compaction_state.h b/be/src/storage/lake/update_compaction_state.h index 674d0ab75f2b8..d5984971c1c49 100644 --- a/be/src/storage/lake/update_compaction_state.h +++ b/be/src/storage/lake/update_compaction_state.h @@ -20,6 +20,7 @@ #include "common/status.h" #include "storage/lake/types_fwd.h" #include "storage/olap_common.h" +#include "storage/tablet_schema.h" namespace starrocks { @@ -34,13 +35,13 @@ class CompactionState { CompactionState(const CompactionState&) = delete; CompactionState& operator=(const CompactionState&) = delete; - Status load_segments(Rowset* rowset, const TabletSchema& tablet_schema, uint32_t segment_id); + Status load_segments(Rowset* rowset, const TabletSchemaCSPtr& tablet_schema, uint32_t segment_id); void release_segments(uint32_t segment_id); std::vector pk_cols; private: - Status _load_segments(Rowset* rowset, const TabletSchema& tablet_schema, uint32_t segment_id); + Status _load_segments(Rowset* rowset, const TabletSchemaCSPtr& tablet_schema, uint32_t segment_id); }; } // namespace lake diff --git a/be/src/storage/lake/update_manager.cpp b/be/src/storage/lake/update_manager.cpp index 63775ff9d310d..87612f0a4d5c2 100644 --- a/be/src/storage/lake/update_manager.cpp +++ b/be/src/storage/lake/update_manager.cpp @@ -70,7 +70,7 @@ Status UpdateManager::publish_primary_key_tablet(const TxnLogPB_OpWrite& op_writ DeferOp release_index_entry([&] { _index_cache.release(index_entry); }); // 2. load rowset update data to cache, get upsert and delete list const uint32_t rowset_id = metadata.next_rowset_id(); - std::unique_ptr tablet_schema = std::make_unique(metadata.schema()); + auto tablet_schema = std::make_shared(metadata.schema()); auto state_entry = _update_state_cache.get_or_create(strings::Substitute("$0_$1", tablet->id(), txn_id)); state_entry->update_expire_time(MonotonicMillis() + get_cache_expire_ms()); // only use state entry once, remove it when publish finish or fail @@ -93,7 +93,7 @@ Status UpdateManager::publish_primary_key_tablet(const TxnLogPB_OpWrite& op_writ if (condition_column < 0) { RETURN_IF_ERROR(_do_update(rowset_id, i, upserts, index, tablet->id(), &new_deletes)); } else { - RETURN_IF_ERROR(_do_update_with_condition(tablet, metadata, op_write, *tablet_schema, rowset_id, i, + RETURN_IF_ERROR(_do_update_with_condition(tablet, metadata, op_write, tablet_schema, rowset_id, i, condition_column, upserts, index, tablet->id(), &new_deletes)); } @@ -171,12 +171,13 @@ Status UpdateManager::_do_update(uint32_t rowset_id, int32_t upsert_idx, const s } Status UpdateManager::_do_update_with_condition(Tablet* tablet, const TabletMetadata& metadata, - const TxnLogPB_OpWrite& op_write, const TabletSchema& tablet_schema, - uint32_t rowset_id, int32_t upsert_idx, int32_t condition_column, + const TxnLogPB_OpWrite& op_write, + const TabletSchemaCSPtr& tablet_schema, uint32_t rowset_id, + int32_t upsert_idx, int32_t condition_column, const std::vector& upserts, PrimaryIndex& index, int64_t tablet_id, DeletesMap* new_deletes) { CHECK(condition_column >= 0); - const auto& tablet_column = tablet_schema.column(condition_column); + const auto& tablet_column = tablet_schema->column(condition_column); std::vector read_column_ids; read_column_ids.push_back(condition_column); @@ -289,7 +290,7 @@ Status UpdateManager::get_rowids_from_pkindex(Tablet* tablet, int64_t base_versi } Status UpdateManager::get_column_values(Tablet* tablet, const TabletMetadata& metadata, - const TxnLogPB_OpWrite& op_write, const TabletSchema& tablet_schema, + const TxnLogPB_OpWrite& op_write, const TabletSchemaCSPtr& tablet_schema, std::vector& column_ids, bool with_default, std::map>& rowids_by_rssid, vector>* columns, @@ -300,7 +301,7 @@ Status UpdateManager::get_column_values(Tablet* tablet, const TabletMetadata& me if (with_default && auto_increment_state == nullptr) { for (auto i = 0; i < column_ids.size(); ++i) { - const TabletColumn& tablet_column = tablet_schema.column(column_ids[i]); + const TabletColumn& tablet_column = tablet_schema->column(column_ids[i]); if (tablet_column.has_default_value()) { const TypeInfoPtr& type_info = get_type_info(tablet_column); std::unique_ptr default_value_iter = @@ -325,7 +326,7 @@ Status UpdateManager::get_column_values(Tablet* tablet, const TabletMetadata& me std::shared_ptr fs; auto fetch_values_from_segment = [&](const std::string& segment_name, uint32_t segment_id, - const TabletSchema* tablet_schema, + const TabletSchemaCSPtr& tablet_schema, const std::vector& rowids) -> Status { std::string path = tablet->segment_location(segment_name); auto segment = Segment::open(fs, path, segment_id, tablet_schema); @@ -357,7 +358,7 @@ Status UpdateManager::get_column_values(Tablet* tablet, const TabletMetadata& me } // use 0 segment_id is safe, because we need not get either delvector or dcg here - RETURN_IF_ERROR(fetch_values_from_segment(rssid_to_path[rssid], 0, &tablet_schema, rowids)); + RETURN_IF_ERROR(fetch_values_from_segment(rssid_to_path[rssid], 0, tablet_schema, rowids)); } if (auto_increment_state != nullptr && with_default) { if (fs == nullptr) { @@ -368,7 +369,7 @@ Status UpdateManager::get_column_values(Tablet* tablet, const TabletMetadata& me const std::vector& rowids = auto_increment_state->rowids; RETURN_IF_ERROR(fetch_values_from_segment(op_write.rowset().segments(segment_id), segment_id, - auto_increment_state->schema.get(), rowids)); + auto_increment_state->schema, rowids)); } cost_str << " [fetch vals by rowid] " << watch.elapsed_time(); VLOG(2) << "UpdateManager get_column_values " << cost_str.str(); @@ -465,7 +466,7 @@ Status UpdateManager::publish_primary_compaction(const TxnLogPB_OpCompaction& op // release index entry but keep it in cache DeferOp release_index_entry([&] { _index_cache.release(index_entry); }); // 2. iterate output rowset, update primary index and generate delvec - std::unique_ptr tablet_schema = std::make_unique(metadata.schema()); + std::shared_ptr tablet_schema = std::make_shared(metadata.schema()); RowsetPtr output_rowset = std::make_shared(tablet, std::make_shared(op_compaction.output_rowset())); auto compaction_state = std::make_unique(output_rowset.get()); @@ -483,7 +484,7 @@ Status UpdateManager::publish_primary_compaction(const TxnLogPB_OpCompaction& op uint32_t max_src_rssid = max_rowset_id + input_rowset->segments_size() - 1; for (size_t i = 0; i < compaction_state->pk_cols.size(); i++) { - RETURN_IF_ERROR(compaction_state->load_segments(output_rowset.get(), *tablet_schema, i)); + RETURN_IF_ERROR(compaction_state->load_segments(output_rowset.get(), tablet_schema, i)); auto& pk_col = compaction_state->pk_cols[i]; total_rows += pk_col->size(); uint32_t rssid = rowset_id + i; diff --git a/be/src/storage/lake/update_manager.h b/be/src/storage/lake/update_manager.h index b9d12f1196842..1991bbd94a01c 100644 --- a/be/src/storage/lake/update_manager.h +++ b/be/src/storage/lake/update_manager.h @@ -70,8 +70,8 @@ class UpdateManager { // get column data by rssid and rowids Status get_column_values(Tablet* tablet, const TabletMetadata& metadata, const TxnLogPB_OpWrite& op_write, - const TabletSchema& tablet_schema, std::vector& column_ids, bool with_default, - std::map>& rowids_by_rssid, + const TabletSchemaCSPtr& tablet_schema, std::vector& column_ids, + bool with_default, std::map>& rowids_by_rssid, vector>* columns, AutoIncrementPartialUpdateState* auto_increment_state = nullptr); // get delvec by version @@ -125,7 +125,7 @@ class UpdateManager { PrimaryIndex& index, int64_t tablet_id, DeletesMap* new_deletes); Status _do_update_with_condition(Tablet* tablet, const TabletMetadata& metadata, const TxnLogPB_OpWrite& op_write, - const TabletSchema& tablet_schema, uint32_t rowset_id, int32_t upsert_idx, + const TabletSchemaCSPtr& tablet_schema, uint32_t rowset_id, int32_t upsert_idx, int32_t condition_column, const std::vector& upserts, PrimaryIndex& index, int64_t tablet_id, DeletesMap* new_deletes); diff --git a/be/src/storage/lake/vertical_compaction_task.cpp b/be/src/storage/lake/vertical_compaction_task.cpp index 1651fdc9d9aff..100f3ac6c8d15 100644 --- a/be/src/storage/lake/vertical_compaction_task.cpp +++ b/be/src/storage/lake/vertical_compaction_task.cpp @@ -133,7 +133,7 @@ Status VerticalCompactionTask::compact_column_group(bool is_key, int column_grou const CancelFunc& cancel_func) { ASSIGN_OR_RETURN(auto chunk_size, calculate_chunk_size_for_column_group(column_group)); - Schema schema = ChunkHelper::convert_schema(*_tablet_schema, column_group); + Schema schema = ChunkHelper::convert_schema(_tablet_schema, column_group); TabletReader reader(*_tablet, _version, schema, _input_rowsets, is_key, mask_buffer); RETURN_IF_ERROR(reader.prepare()); TabletReaderParams reader_params; @@ -166,7 +166,7 @@ Status VerticalCompactionTask::compact_column_group(bool is_key, int column_grou return st; } - ChunkHelper::padding_char_columns(char_field_indexes, schema, *_tablet_schema, chunk.get()); + ChunkHelper::padding_char_columns(char_field_indexes, schema, _tablet_schema, chunk.get()); RETURN_IF_ERROR(writer->write_columns(*chunk, column_group, is_key)); chunk->reset(); diff --git a/be/src/storage/local_tablet_reader.cpp b/be/src/storage/local_tablet_reader.cpp index d65cdd738e4ba..6302b8f0da671 100644 --- a/be/src/storage/local_tablet_reader.cpp +++ b/be/src/storage/local_tablet_reader.cpp @@ -90,7 +90,7 @@ Status LocalTabletReader::multi_get(const Chunk& keys, const std::vectortablet_schema(); std::vector value_column_ids; for (const auto& name : value_columns) { - auto cid = tablet_schema.field_index(name); + auto cid = tablet_schema->field_index(name); if (cid == -1) { return Status::InvalidArgument(strings::Substitute("multi_get value_column $0 not found", name)); } @@ -111,14 +111,14 @@ Status LocalTabletReader::multi_get(const Chunk& keys, const std::vectortablet_schema(); vector pk_columns; - for (size_t i = 0; i < tablet_schema.num_key_columns(); i++) { + for (size_t i = 0; i < tablet_schema->num_key_columns(); i++) { pk_columns.push_back((uint32_t)i); } std::unique_ptr pk_column; - if (!PrimaryKeyEncoder::create_column(*tablet_schema.schema(), &pk_column).ok()) { + if (!PrimaryKeyEncoder::create_column(*tablet_schema->schema(), &pk_column).ok()) { CHECK(false) << "create column for primary key encoder failed"; } - PrimaryKeyEncoder::encode(*tablet_schema.schema(), keys, 0, keys.num_rows(), pk_column.get()); + PrimaryKeyEncoder::encode(*tablet_schema->schema(), keys, 0, keys.num_rows(), pk_column.get()); // search pks in pk index to get rowids EditVersion edit_version; @@ -163,7 +163,7 @@ StatusOr LocalTabletReader::scan(const std::vector& predicates) { TabletReaderParams tablet_reader_params; tablet_reader_params.predicates = predicates; - auto& full_schema = *_tablet->tablet_schema().schema(); + auto& full_schema = *_tablet->tablet_schema()->schema(); vector column_ids; for (auto& cname : value_columns) { auto idx = full_schema.get_field_index_by_name(cname); @@ -194,19 +194,19 @@ Status handle_tablet_multi_get_rpc(const PTabletReaderMultiGetRequest& request, const auto& tablet_schema = tablet->tablet_schema(); const auto& keys_pb = request.keys(); vector key_column_ids; - for (size_t i = 0; i < tablet_schema.num_key_columns(); i++) { + for (size_t i = 0; i < tablet_schema->num_key_columns(); i++) { key_column_ids.push_back(i); } - Schema key_schema(tablet_schema.schema(), key_column_ids); + Schema key_schema(tablet_schema->schema(), key_column_ids); std::vector value_column_ids; for (const auto& name : value_columns) { - auto cid = tablet_schema.field_index(name); + auto cid = tablet_schema->field_index(name); if (cid == -1) { return Status::InvalidArgument(strings::Substitute("multi_get value_column $0 not found", name)); } value_column_ids.push_back(cid); } - Schema values_schema(tablet->tablet_schema().schema(), value_column_ids); + Schema values_schema(tablet->tablet_schema()->schema(), value_column_ids); auto keys_st = serde::deserialize_chunk_pb_with_schema(key_schema, keys_pb.data()); if (!keys_st.ok()) { return keys_st.status(); diff --git a/be/src/storage/memtable.cpp b/be/src/storage/memtable.cpp index cf14e90b15581..3fd05f997ad11 100644 --- a/be/src/storage/memtable.cpp +++ b/be/src/storage/memtable.cpp @@ -35,8 +35,9 @@ namespace starrocks { // TODO(cbl): move to common space latter static const string LOAD_OP_COLUMN = "__op"; -Schema MemTable::convert_schema(const TabletSchema* tablet_schema, const std::vector* slot_descs) { - Schema schema = ChunkHelper::convert_schema(*tablet_schema); +Schema MemTable::convert_schema(const TabletSchemaCSPtr& tablet_schema, + const std::vector* slot_descs) { + Schema schema = ChunkHelper::convert_schema(tablet_schema); if (tablet_schema->keys_type() == KeysType::PRIMARY_KEYS && slot_descs != nullptr && slot_descs->back()->col_name() == LOAD_OP_COLUMN) { // load slots have __op field, so add to _vectorized_schema diff --git a/be/src/storage/memtable.h b/be/src/storage/memtable.h index 05da502aeb9f1..e7f6f3a25e7fd 100644 --- a/be/src/storage/memtable.h +++ b/be/src/storage/memtable.h @@ -68,7 +68,8 @@ class MemTable { _partial_schema_with_sort_key = partial_schema_with_sort_key; } - static Schema convert_schema(const TabletSchema* tablet_schema, const std::vector* slot_descs); + static Schema convert_schema(const TabletSchemaCSPtr& tablet_schema, + const std::vector* slot_descs); ChunkPtr get_result_chunk() { return _result_chunk; } diff --git a/be/src/storage/meta_reader.cpp b/be/src/storage/meta_reader.cpp index 298e2326e5ff9..07f5b6de12f45 100644 --- a/be/src/storage/meta_reader.cpp +++ b/be/src/storage/meta_reader.cpp @@ -159,7 +159,8 @@ Status SegmentMetaCollecter::_init_return_column_iterators() { if (_params->read_page[i]) { auto cid = _params->cids[i]; if (_column_iterators[cid] == nullptr) { - ASSIGN_OR_RETURN(_column_iterators[cid], _segment->new_column_iterator(cid)); + ASSIGN_OR_RETURN(_column_iterators[cid], + _segment->new_column_iterator(cid, nullptr, _params->tablet_schema)); ColumnIteratorOptions iter_opts; iter_opts.check_dict_encoding = true; diff --git a/be/src/storage/meta_reader.h b/be/src/storage/meta_reader.h index 8de7465e91e73..e3402dc757dfe 100644 --- a/be/src/storage/meta_reader.h +++ b/be/src/storage/meta_reader.h @@ -57,6 +57,8 @@ struct SegmentMetaCollecterParams { std::vector read_page; std::vector field_type; int32_t max_cid; + bool use_page_cache; + TabletSchemaCSPtr tablet_schema; }; // MetaReader will implements diff --git a/be/src/storage/metadata_util.cpp b/be/src/storage/metadata_util.cpp index 39d390294d07b..cec9ab324debd 100644 --- a/be/src/storage/metadata_util.cpp +++ b/be/src/storage/metadata_util.cpp @@ -27,7 +27,7 @@ namespace starrocks { // Old version StarRocks use `TColumnType` to save type info, convert it into `TTypeDesc`. // NOTE: This is only used for some legacy UT -static void convert_to_new_version(TColumn* tcolumn) { +void convert_to_new_version(TColumn* tcolumn) { if (!tcolumn->__isset.type_desc) { tcolumn->__set_index_len(tcolumn->column_type.index_len); @@ -43,7 +43,6 @@ static void convert_to_new_version(TColumn* tcolumn) { tcolumn->__isset.type_desc = true; } } - static StorageAggregateType t_aggregation_type_to_field_aggregation_method(TAggregationType::type agg_type) { switch (agg_type) { case TAggregationType::NONE: @@ -152,7 +151,7 @@ static Status type_desc_to_pb(const std::vector& types, int* index, C return Status::InternalError("Unreachable path"); } -static Status t_column_to_pb_column(int32_t unique_id, const TColumn& t_column, ColumnPB* column_pb) { +Status t_column_to_pb_column(int32_t unique_id, const TColumn& t_column, ColumnPB* column_pb) { DCHECK(t_column.__isset.type_desc); const std::vector& types = t_column.type_desc.types; int index = 0; @@ -165,7 +164,7 @@ static Status t_column_to_pb_column(int32_t unique_id, const TColumn& t_column, column_pb->set_name(t_column.column_name); column_pb->set_is_key(t_column.is_key); column_pb->set_is_nullable(t_column.is_allow_null); - + column_pb->set_has_bitmap_index(t_column.has_bitmap_index); if (t_column.is_key) { auto agg_method = STORAGE_AGGREGATE_NONE; column_pb->set_aggregation(get_string_by_aggregation_type(agg_method)); @@ -186,10 +185,6 @@ static Status t_column_to_pb_column(int32_t unique_id, const TColumn& t_column, column_pb->set_is_bf_column(t_column.is_bloom_filter_column); } - if (t_column.__isset.is_auto_increment) { - column_pb->set_is_auto_increment(t_column.is_auto_increment); - } - return Status::OK(); } @@ -242,7 +237,13 @@ Status convert_t_schema_to_pb_schema(const TTabletSchema& tablet_schema, uint32_ bool has_bf_columns = false; for (TColumn tcolumn : tablet_schema.columns) { convert_to_new_version(&tcolumn); - uint32_t col_unique_id = col_ordinal_to_unique_id.at(col_ordinal++); + uint32_t col_unique_id; + if (tcolumn.col_unique_id >= 0) { + col_unique_id = tcolumn.col_unique_id; + } else { + col_unique_id = col_ordinal_to_unique_id.at(col_ordinal); + } + col_ordinal++; ColumnPB* column = schema->add_column(); RETURN_IF_ERROR(t_column_to_pb_column(col_unique_id, tcolumn, column)); diff --git a/be/src/storage/metadata_util.h b/be/src/storage/metadata_util.h index ec7de331c38ec..9dd0aac564aab 100644 --- a/be/src/storage/metadata_util.h +++ b/be/src/storage/metadata_util.h @@ -14,10 +14,15 @@ #pragma once +#include +#include + #include #include "common/status.h" #include "gen_cpp/Types_types.h" +#include "olap_common.h" +#include "types/logical_type.h" namespace starrocks { @@ -25,8 +30,17 @@ class TTabletSchema; class TabletSchemaPB; enum RowsetTypePB : int; +enum class FieldTypeVersion { + kV1, + kV2, +}; + Status convert_t_schema_to_pb_schema(const TTabletSchema& tablet_schema, uint32_t next_unique_id, const std::unordered_map& col_ordinal_to_unique_id, TabletSchemaPB* schema, TCompressionType::type compression_type); -} // namespace starrocks +void convert_to_new_version(TColumn* tcolumn); + +Status t_column_to_pb_column(int32_t unique_id, const TColumn& t_column, ColumnPB* column_pb); + +} // namespace starrocks \ No newline at end of file diff --git a/be/src/storage/olap_meta_reader.cpp b/be/src/storage/olap_meta_reader.cpp index 2076caec9f248..ccf35b7bbcb39 100644 --- a/be/src/storage/olap_meta_reader.cpp +++ b/be/src/storage/olap_meta_reader.cpp @@ -72,7 +72,7 @@ Status OlapMetaReader::_build_collect_context(const OlapMetaReaderParams& read_p } // get column type - LogicalType type = _tablet->tablet_schema().column(index).type(); + LogicalType type = _tablet->tablet_schema()->column(index).type(); _collect_context.seg_collecter_params.field_type.emplace_back(type); // get collect field @@ -94,6 +94,7 @@ Status OlapMetaReader::_build_collect_context(const OlapMetaReaderParams& read_p } _has_count_agg |= (collect_field == "count"); } + _collect_context.seg_collecter_params.tablet_schema = read_params.tablet->tablet_schema(); return Status::OK(); } diff --git a/be/src/storage/persistent_index.cpp b/be/src/storage/persistent_index.cpp index 092593982b8f7..d1a2a7c34948b 100644 --- a/be/src/storage/persistent_index.cpp +++ b/be/src/storage/persistent_index.cpp @@ -3087,12 +3087,12 @@ Status PersistentIndex::load_from_tablet(Tablet* tablet) { } } - const TabletSchema& tablet_schema = tablet->tablet_schema(); - vector pk_columns(tablet_schema.num_key_columns()); - for (auto i = 0; i < tablet_schema.num_key_columns(); i++) { + auto tablet_schema_ptr = tablet->thread_safe_get_tablet_schema(); + vector pk_columns(tablet_schema_ptr->num_key_columns()); + for (auto i = 0; i < tablet_schema_ptr->num_key_columns(); i++) { pk_columns[i] = (ColumnId)i; } - auto pkey_schema = ChunkHelper::convert_schema(tablet_schema, pk_columns); + auto pkey_schema = ChunkHelper::convert_schema(tablet_schema_ptr, pk_columns); size_t fix_size = PrimaryKeyEncoder::get_encoded_fixed_size(pkey_schema); // Init PersistentIndex diff --git a/be/src/storage/predicate_parser.cpp b/be/src/storage/predicate_parser.cpp index cecbef5e95e6b..c92273810c34c 100644 --- a/be/src/storage/predicate_parser.cpp +++ b/be/src/storage/predicate_parser.cpp @@ -27,24 +27,24 @@ namespace starrocks { bool PredicateParser::can_pushdown(const ColumnPredicate* predicate) const { - RETURN_IF(predicate->column_id() >= _schema.num_columns(), false); - const TabletColumn& column = _schema.column(predicate->column_id()); - return _schema.keys_type() == KeysType::PRIMARY_KEYS || + RETURN_IF(predicate->column_id() >= _schema->num_columns(), false); + const TabletColumn& column = _schema->column(predicate->column_id()); + return _schema->keys_type() == KeysType::PRIMARY_KEYS || column.aggregation() == StorageAggregateType::STORAGE_AGGREGATE_NONE; } bool PredicateParser::can_pushdown(const SlotDescriptor* slot_desc) const { - const size_t index = _schema.field_index(slot_desc->col_name()); - CHECK(index <= _schema.num_columns()); - const TabletColumn& column = _schema.column(index); - return _schema.keys_type() == KeysType::PRIMARY_KEYS || + const size_t index = _schema->field_index(slot_desc->col_name()); + CHECK(index <= _schema->num_columns()); + const TabletColumn& column = _schema->column(index); + return _schema->keys_type() == KeysType::PRIMARY_KEYS || column.aggregation() == StorageAggregateType::STORAGE_AGGREGATE_NONE; } ColumnPredicate* PredicateParser::parse_thrift_cond(const TCondition& condition) const { - const size_t index = _schema.field_index(condition.column_name); - RETURN_IF(index >= _schema.num_columns(), nullptr); - const TabletColumn& col = _schema.column(index); + const size_t index = _schema->field_index(condition.column_name); + RETURN_IF(index >= _schema->num_columns(), nullptr); + const TabletColumn& col = _schema->column(index); auto precision = col.precision(); auto scale = col.scale(); auto type = col.type(); @@ -87,9 +87,9 @@ ColumnPredicate* PredicateParser::parse_thrift_cond(const TCondition& condition) StatusOr PredicateParser::parse_expr_ctx(const SlotDescriptor& slot_desc, RuntimeState* state, ExprContext* expr_ctx) const { - const size_t column_id = _schema.field_index(slot_desc.col_name()); - RETURN_IF(column_id >= _schema.num_columns(), nullptr); - const TabletColumn& col = _schema.column(column_id); + const size_t column_id = _schema->field_index(slot_desc.col_name()); + RETURN_IF(column_id >= _schema->num_columns(), nullptr); + const TabletColumn& col = _schema->column(column_id); auto precision = col.precision(); auto scale = col.scale(); auto type = col.type(); @@ -98,7 +98,7 @@ StatusOr PredicateParser::parse_expr_ctx(const SlotDescriptor& } uint32_t PredicateParser::column_id(const SlotDescriptor& slot_desc) { - return _schema.field_index(slot_desc.col_name()); + return _schema->field_index(slot_desc.col_name()); } } // namespace starrocks diff --git a/be/src/storage/predicate_parser.h b/be/src/storage/predicate_parser.h index 362f04d6e2185..497d834e2fcd9 100644 --- a/be/src/storage/predicate_parser.h +++ b/be/src/storage/predicate_parser.h @@ -15,8 +15,10 @@ #pragma once #include +#include #include "common/statusor.h" +#include "tablet_schema.h" namespace starrocks { @@ -30,7 +32,7 @@ class ColumnPredicate; class PredicateParser { public: - explicit PredicateParser(const TabletSchema& schema) : _schema(schema) {} + explicit PredicateParser(TabletSchemaCSPtr schema) : _schema(std::move(schema)) {} // check if an expression can be pushed down to the storage level bool can_pushdown(const ColumnPredicate* predicate) const; @@ -47,7 +49,7 @@ class PredicateParser { uint32_t column_id(const SlotDescriptor& slot_desc); private: - const TabletSchema& _schema; + const TabletSchemaCSPtr _schema; }; } // namespace starrocks diff --git a/be/src/storage/primary_index.cpp b/be/src/storage/primary_index.cpp index 439d621ec2ffb..47aacb6b79df1 100644 --- a/be/src/storage/primary_index.cpp +++ b/be/src/storage/primary_index.cpp @@ -1017,12 +1017,12 @@ Status PrimaryIndex::_do_load(Tablet* tablet) { MonotonicStopWatch timer; timer.start(); - const TabletSchema& tablet_schema = tablet->tablet_schema(); - vector pk_columns(tablet_schema.num_key_columns()); - for (auto i = 0; i < tablet_schema.num_key_columns(); i++) { + const TabletSchemaCSPtr tablet_schema_ptr = tablet->thread_safe_get_tablet_schema(); + vector pk_columns(tablet_schema_ptr->num_key_columns()); + for (auto i = 0; i < tablet_schema_ptr->num_key_columns(); i++) { pk_columns[i] = (ColumnId)i; } - auto pkey_schema = ChunkHelper::convert_schema(tablet_schema, pk_columns); + auto pkey_schema = ChunkHelper::convert_schema(tablet_schema_ptr, pk_columns); _set_schema(pkey_schema); // load persistent index if enable persistent index meta diff --git a/be/src/storage/push_handler.cpp b/be/src/storage/push_handler.cpp index 297a9b62ff5de..ca9dc0b2a3e18 100644 --- a/be/src/storage/push_handler.cpp +++ b/be/src/storage/push_handler.cpp @@ -102,8 +102,14 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR DeletePredicatePB del_pred; DeleteConditionHandler del_cond_handler; tablet_var.tablet->obtain_header_rdlock(); - res = del_cond_handler.generate_delete_predicate(tablet_var.tablet->tablet_schema(), - request.delete_conditions, &del_pred); + auto tablet_schema = TabletSchema::copy(tablet_var.tablet->thread_safe_get_tablet_schema()); + if (!request.columns_desc.empty() && request.columns_desc[0].col_unique_id >= 0) { + tablet_schema->clear_columns(); + for (const auto& column_desc : request.columns_desc) { + tablet_schema->append_column(TabletColumn(column_desc)); + } + } + res = del_cond_handler.generate_delete_predicate(*tablet_schema, request.delete_conditions, &del_pred); del_preds.push(del_pred); tablet_var.tablet->release_header_lock(); if (!res.ok()) { @@ -114,12 +120,20 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR } } + auto tablet_schema = std::shared_ptr(TabletSchema::copy(tablet_vars->at(0).tablet->tablet_schema())); + if (!request.columns_desc.empty() && request.columns_desc[0].col_unique_id >= 0) { + tablet_schema->clear_columns(); + for (const auto& column_desc : request.columns_desc) { + tablet_schema->append_column(TabletColumn(column_desc)); + } + } + Status st = Status::OK(); if (push_type == PUSH_NORMAL_V2) { - st = _load_convert(tablet_vars->at(0).tablet, &(tablet_vars->at(0).rowset_to_add)); + st = _load_convert(tablet_vars->at(0).tablet, &(tablet_vars->at(0).rowset_to_add), tablet_schema); } else { DCHECK_EQ(push_type, PUSH_FOR_DELETE); - st = _delete_convert(tablet_vars->at(0).tablet, &(tablet_vars->at(0).rowset_to_add)); + st = _delete_convert(tablet_vars->at(0).tablet, &(tablet_vars->at(0).rowset_to_add), tablet_schema); } if (!st.ok()) { @@ -180,7 +194,8 @@ void PushHandler::_get_tablet_infos(const std::vector& tablet_vars, } } -Status PushHandler::_delete_convert(const TabletSharedPtr& cur_tablet, RowsetSharedPtr* cur_rowset) { +Status PushHandler::_delete_convert(const TabletSharedPtr& cur_tablet, RowsetSharedPtr* cur_rowset, + const TabletSchemaCSPtr& tablet_schema) { Status st = Status::OK(); PUniqueId load_id; load_id.set_hi(0); @@ -199,7 +214,7 @@ Status PushHandler::_delete_convert(const TabletSharedPtr& cur_tablet, RowsetSha context.partition_id = _request.partition_id; context.tablet_schema_hash = cur_tablet->schema_hash(); context.rowset_path_prefix = cur_tablet->schema_hash_path(); - context.tablet_schema = &cur_tablet->tablet_schema(); + context.tablet_schema = tablet_schema; context.rowset_state = PREPARED; context.txn_id = _request.transaction_id; context.load_id = load_id; @@ -236,7 +251,8 @@ Status PushHandler::_delete_convert(const TabletSharedPtr& cur_tablet, RowsetSha return st; } -Status PushHandler::_load_convert(const TabletSharedPtr& cur_tablet, RowsetSharedPtr* cur_rowset) { +Status PushHandler::_load_convert(const TabletSharedPtr& cur_tablet, RowsetSharedPtr* cur_rowset, + const TabletSchemaCSPtr& tablet_schema) { Status st; size_t num_rows = 0; PUniqueId load_id; @@ -247,7 +263,7 @@ Status PushHandler::_load_convert(const TabletSharedPtr& cur_tablet, RowsetShare // 1. init RowsetBuilder of cur_tablet for current push VLOG(3) << "init rowset builder. tablet=" << cur_tablet->full_name() - << ", block_row_size=" << cur_tablet->num_rows_per_row_block(); + << ", block_row_size=" << tablet_schema->num_rows_per_row_block(); RowsetWriterContext context; context.rowset_id = StorageEngine::instance()->next_rowset_id(); context.tablet_uid = cur_tablet->tablet_uid(); @@ -255,7 +271,7 @@ Status PushHandler::_load_convert(const TabletSharedPtr& cur_tablet, RowsetShare context.partition_id = _request.partition_id; context.tablet_schema_hash = cur_tablet->schema_hash(); context.rowset_path_prefix = cur_tablet->schema_hash_path(); - context.tablet_schema = &(cur_tablet->tablet_schema()); + context.tablet_schema = tablet_schema; context.rowset_state = PREPARED; context.txn_id = _request.transaction_id; context.load_id = load_id; @@ -297,7 +313,7 @@ Status PushHandler::_load_convert(const TabletSharedPtr& cur_tablet, RowsetShare // read data from broker and write into Rowset of cur_tablet VLOG(3) << "start to convert etl file to delta."; - auto schema = ChunkHelper::convert_schema(cur_tablet->tablet_schema()); + auto schema = ChunkHelper::convert_schema(tablet_schema); ChunkPtr chunk = ChunkHelper::new_chunk(schema, 0); while (!reader->eof()) { st = reader->next_chunk(&chunk); diff --git a/be/src/storage/push_handler.h b/be/src/storage/push_handler.h index 3737e32a2deba..8f181913136d0 100644 --- a/be/src/storage/push_handler.h +++ b/be/src/storage/push_handler.h @@ -51,8 +51,10 @@ class PushHandler { void _get_tablet_infos(const std::vector& tablet_infos, std::vector* tablet_info_vec); - Status _load_convert(const TabletSharedPtr& cur_tablet, RowsetSharedPtr* cur_rowset); - Status _delete_convert(const TabletSharedPtr& cur_tablet, RowsetSharedPtr* cur_rowset); + Status _load_convert(const TabletSharedPtr& cur_tablet, RowsetSharedPtr* cur_rowset, + const TabletSchemaCSPtr& tablet_schema); + Status _delete_convert(const TabletSharedPtr& cur_tablet, RowsetSharedPtr* cur_rowset, + const TabletSchemaCSPtr& tablet_schema); private: // mainly tablet_id, version and delta file path diff --git a/be/src/storage/rowset/horizontal_update_rowset_writer.cpp b/be/src/storage/rowset/horizontal_update_rowset_writer.cpp index 66a896abba910..9da336a3de4e9 100644 --- a/be/src/storage/rowset/horizontal_update_rowset_writer.cpp +++ b/be/src/storage/rowset/horizontal_update_rowset_writer.cpp @@ -42,7 +42,7 @@ StatusOr> HorizontalUpdateRowsetWriter::_create_u std::lock_guard l(_lock); std::string path = Rowset::segment_upt_file_path(_context.rowset_path_prefix, _context.rowset_id, _num_uptfile); ASSIGN_OR_RETURN(auto wfile, _fs->new_writable_file(path)); - const auto* schema = _context.tablet_schema; + const auto schema = _context.tablet_schema; auto segment_writer = std::make_unique(std::move(wfile), _num_uptfile, schema, _writer_options); RETURN_IF_ERROR(segment_writer->init()); return std::move(segment_writer); diff --git a/be/src/storage/rowset/rowset.cpp b/be/src/storage/rowset/rowset.cpp index 4f7efb8090b61..b2f092239ff7a 100644 --- a/be/src/storage/rowset/rowset.cpp +++ b/be/src/storage/rowset/rowset.cpp @@ -32,8 +32,6 @@ // specific language governing permissions and limitations // under the License. -#include "storage/rowset/rowset.h" - #include #include @@ -64,11 +62,12 @@ namespace starrocks { -Rowset::Rowset(const TabletSchema* schema, std::string rowset_path, RowsetMetaSharedPtr rowset_meta) +Rowset::Rowset(const TabletSchemaCSPtr& schema, std::string rowset_path, RowsetMetaSharedPtr rowset_meta) : _schema(schema), _rowset_path(std::move(rowset_path)), _rowset_meta(std::move(rowset_meta)), _refs_by_reader(0) { + _schema = _rowset_meta->tablet_schema() ? _rowset_meta->tablet_schema() : schema; _keys_type = _schema->keys_type(); MEM_TRACKER_SAFE_CONSUME(GlobalEnv::GetInstance()->rowset_metadata_mem_tracker(), _mem_usage()); } @@ -552,7 +551,6 @@ StatusOr Rowset::new_iterator(const Schema& schema, const Rows Status Rowset::get_segment_iterators(const Schema& schema, const RowsetReadOptions& options, std::vector* segment_iterators) { RowsetReleaseGuard guard(shared_from_this()); - RETURN_IF_ERROR(load()); SegmentReadOptions seg_options; @@ -569,6 +567,7 @@ Status Rowset::get_segment_iterators(const Schema& schema, const RowsetReadOptio seg_options.unused_output_column_ids = options.unused_output_column_ids; seg_options.runtime_range_pruner = options.runtime_range_pruner; seg_options.column_access_paths = options.column_access_paths; + seg_options.tablet_schema = options.tablet_schema; if (options.delete_predicates != nullptr) { seg_options.delete_predicates = options.delete_predicates->get_predicates(end_version()); } @@ -801,7 +800,7 @@ Status Rowset::verify() { order_columns = key_columns; is_pk_ordered = _schema->keys_type() == PRIMARY_KEYS; } - Schema order_schema = ChunkHelper::convert_schema(*_schema, order_columns); + Schema order_schema = ChunkHelper::convert_schema(_schema, order_columns); RowsetReadOptions rs_opts; OlapReaderStatistics stats; rs_opts.sorted = false; diff --git a/be/src/storage/rowset/rowset.h b/be/src/storage/rowset/rowset.h index 6fe3de9632cba..acd44a01d42c6 100644 --- a/be/src/storage/rowset/rowset.h +++ b/be/src/storage/rowset/rowset.h @@ -37,6 +37,7 @@ #include #include #include +#include #include #include "common/statusor.h" @@ -132,13 +133,13 @@ class RowsetStateMachine { class Rowset : public std::enable_shared_from_this { public: - Rowset(const TabletSchema* schema, std::string rowset_path, RowsetMetaSharedPtr rowset_meta); + Rowset(const TabletSchemaCSPtr&, std::string rowset_path, RowsetMetaSharedPtr rowset_meta); Rowset(const Rowset&) = delete; const Rowset& operator=(const Rowset&) = delete; virtual ~Rowset(); - static std::shared_ptr create(const TabletSchema* schema, std::string rowset_path, + static std::shared_ptr create(const TabletSchemaCSPtr& schema, std::string rowset_path, RowsetMetaSharedPtr rowset_meta) { return std::make_shared(schema, std::move(rowset_path), std::move(rowset_meta)); } @@ -154,8 +155,9 @@ class Rowset : public std::enable_shared_from_this { Status reload_segment(int32_t segment_id); int64_t total_segment_data_size(); - const TabletSchema& schema() const { return *_schema; } - void set_schema(const TabletSchema* schema) { _schema = schema; } + const TabletSchema& schema_ref() const { return *_schema; } + const TabletSchemaCSPtr& schema() const { return _schema; } + void set_schema(const TabletSchemaCSPtr& schema) { _schema = schema; } StatusOr new_iterator(const Schema& schema, const RowsetReadOptions& options); @@ -363,7 +365,7 @@ class Rowset : public std::enable_shared_from_this { // allow subclass to add custom logic when rowset is being published virtual void make_visible_extra(Version version) {} - const TabletSchema* _schema; + TabletSchemaCSPtr _schema; std::string _rowset_path; RowsetMetaSharedPtr _rowset_meta; @@ -396,5 +398,6 @@ class RowsetReleaseGuard { private: std::shared_ptr _rowset; }; +using TabletSchemaSPtr = std::shared_ptr; } // namespace starrocks diff --git a/be/src/storage/rowset/rowset_factory.cpp b/be/src/storage/rowset/rowset_factory.cpp index e67c5bf94bede..9f50ce22f63c8 100644 --- a/be/src/storage/rowset/rowset_factory.cpp +++ b/be/src/storage/rowset/rowset_factory.cpp @@ -44,7 +44,7 @@ namespace starrocks { -Status RowsetFactory::create_rowset(const TabletSchema* schema, const std::string& rowset_path, +Status RowsetFactory::create_rowset(const TabletSchemaCSPtr& schema, const std::string& rowset_path, const RowsetMetaSharedPtr& rowset_meta, RowsetSharedPtr* rowset) { *rowset = Rowset::create(schema, rowset_path, rowset_meta); RETURN_IF_ERROR((*rowset)->init()); diff --git a/be/src/storage/rowset/rowset_factory.h b/be/src/storage/rowset/rowset_factory.h index 929d68951a0ae..f835ac7bbc57c 100644 --- a/be/src/storage/rowset/rowset_factory.h +++ b/be/src/storage/rowset/rowset_factory.h @@ -47,7 +47,7 @@ class RowsetFactory { public: // return OK on success and set inited rowset in `*rowset`. // return error if failed to create or init rowset. - static Status create_rowset(const TabletSchema* schema, const std::string& rowset_path, + static Status create_rowset(const TabletSchemaCSPtr& schema, const std::string& rowset_path, const RowsetMetaSharedPtr& rowset_meta, RowsetSharedPtr* rowset); // create and init rowset writer. diff --git a/be/src/storage/rowset/rowset_meta.h b/be/src/storage/rowset/rowset_meta.h index d52b59d140355..2dd37893a6675 100644 --- a/be/src/storage/rowset/rowset_meta.h +++ b/be/src/storage/rowset/rowset_meta.h @@ -45,6 +45,7 @@ #include "json2pb/json_to_pb.h" #include "json2pb/pb_to_json.h" #include "storage/olap_common.h" +#include "storage/tablet_schema.h" namespace starrocks { @@ -173,10 +174,6 @@ class RowsetMeta { SegmentsOverlapPB segments_overlap() const { return _rowset_meta_pb->segments_overlap_pb(); } - void set_segments_overlap_pb(SegmentsOverlapPB overlap) { - return _rowset_meta_pb->set_segments_overlap_pb(overlap); - } - // return true if segments in this rowset has overlapping data. // this is not same as `segments_overlap()` method. // `segments_overlap()` only return the value of "segments_overlap" field in rowset meta, @@ -209,6 +206,10 @@ class RowsetMeta { uint32_t get_rowset_seg_id() const { return _rowset_meta_pb->rowset_seg_id(); } + void set_segments_overlap_pb(SegmentsOverlapPB overlap) { + return _rowset_meta_pb->set_segments_overlap_pb(overlap); + } + void set_rowset_seg_id(uint32_t id) { _rowset_meta_pb->set_rowset_seg_id(id); } uint32_t get_num_delete_files() const { return _rowset_meta_pb->num_delete_files(); } @@ -217,6 +218,15 @@ class RowsetMeta { const RowsetMetaPB& get_meta_pb() const { return *_rowset_meta_pb; } + void set_tablet_schema(const TabletSchemaCSPtr& tablet_schema_ptr) { + TabletSchemaPB* ts_pb = _rowset_meta_pb->mutable_tablet_schema(); + tablet_schema_ptr->to_schema_pb(ts_pb); + CHECK(_schema == nullptr); + _schema = TabletSchemaCSPtr(TabletSchema::copy(tablet_schema_ptr)); + } + + const TabletSchemaCSPtr tablet_schema() { return _schema; } + void set_partial_schema_change(bool partial_schema_change) { _rowset_meta_pb->set_partial_schema_change(partial_schema_change); } @@ -234,6 +244,9 @@ class RowsetMeta { } else { _rowset_id.init(_rowset_meta_pb->rowset_id()); } + if (_rowset_meta_pb->has_tablet_schema()) { + _schema = TabletSchema::create(_rowset_meta_pb->tablet_schema()); + } } int64_t _calc_mem_usage() const { @@ -261,6 +274,7 @@ class RowsetMeta { std::unique_ptr _rowset_meta_pb; RowsetId _rowset_id; bool _is_removed_from_rowset_meta = false; + TabletSchemaCSPtr _schema = nullptr; }; } // namespace starrocks diff --git a/be/src/storage/rowset/rowset_options.h b/be/src/storage/rowset/rowset_options.h index ffef412872797..04678270867fc 100644 --- a/be/src/storage/rowset/rowset_options.h +++ b/be/src/storage/rowset/rowset_options.h @@ -23,6 +23,7 @@ #include "storage/olap_common.h" #include "storage/olap_runtime_range_pruner.h" #include "storage/seek_range.h" +#include "storage/tablet_schema.h" namespace starrocks { class Conditions; @@ -57,7 +58,7 @@ class RowsetReadOptions { const DeletePredicates* delete_predicates = nullptr; - const TabletSchema* tablet_schema = nullptr; + TabletSchemaCSPtr tablet_schema = nullptr; bool is_primary_keys = false; int64_t version = 0; diff --git a/be/src/storage/rowset/rowset_writer.cpp b/be/src/storage/rowset/rowset_writer.cpp index 1ee2a6b394212..ba65bc7c57d25 100644 --- a/be/src/storage/rowset/rowset_writer.cpp +++ b/be/src/storage/rowset/rowset_writer.cpp @@ -438,7 +438,7 @@ StatusOr> HorizontalRowsetWriter::_create_segment path = Rowset::segment_file_path(_context.rowset_path_prefix, _context.rowset_id, _num_segment); } ASSIGN_OR_RETURN(auto wfile, _fs->new_writable_file(path)); - const auto* schema = _context.tablet_schema; + const auto schema = _context.tablet_schema; auto segment_writer = std::make_unique(std::move(wfile), _num_segment, schema, _writer_options); RETURN_IF_ERROR(segment_writer->init()); ++_num_segment; @@ -683,7 +683,7 @@ Status HorizontalRowsetWriter::_final_merge() { _context.tablet_schema->num_columns(), _context.tablet_schema->sort_key_idxes(), config::vertical_compaction_max_columns_per_group, &column_groups); } - auto schema = ChunkHelper::convert_schema(*_context.tablet_schema, column_groups[0]); + auto schema = ChunkHelper::convert_schema(_context.tablet_schema, column_groups[0]); if (!_context.merge_condition.empty()) { for (int i = _context.tablet_schema->num_key_columns(); i < _context.tablet_schema->num_columns(); ++i) { if (_context.tablet_schema->schema()->field(i)->name() == _context.merge_condition) { @@ -767,7 +767,7 @@ Status HorizontalRowsetWriter::_final_merge() { if (st.is_end_of_file()) { break; } else if (st.ok()) { - ChunkHelper::padding_char_columns(char_field_indexes, schema, *_context.tablet_schema, chunk); + ChunkHelper::padding_char_columns(char_field_indexes, schema, _context.tablet_schema, chunk); total_rows += chunk->num_rows(); total_chunk++; if (auto st = _vertical_rowset_writer->add_columns(*chunk, column_groups[0], true); !st.ok()) { @@ -794,7 +794,7 @@ Status HorizontalRowsetWriter::_final_merge() { seg_iterators.clear(); - auto schema = ChunkHelper::convert_schema(*_context.tablet_schema, column_groups[i]); + auto schema = ChunkHelper::convert_schema(_context.tablet_schema, column_groups[i]); for (const auto& segment : segments) { auto res = segment->new_iterator(schema, seg_options); @@ -834,7 +834,7 @@ Status HorizontalRowsetWriter::_final_merge() { if (st.is_end_of_file()) { break; } else if (st.ok()) { - ChunkHelper::padding_char_columns(char_field_indexes, schema, *_context.tablet_schema, chunk); + ChunkHelper::padding_char_columns(char_field_indexes, schema, _context.tablet_schema, chunk); if (auto st = _vertical_rowset_writer->add_columns(*chunk, column_groups[i], false); !st.ok()) { LOG(WARNING) << "writer add_columns error. tablet=" << _context.tablet_id << ", err=" << st; return st; @@ -865,7 +865,7 @@ Status HorizontalRowsetWriter::_final_merge() { << " chunk=" << total_chunk << " bytes=" << PrettyPrinter::print(total_data_size(), TUnit::UNIT) << ") duration: " << timer.elapsed_time() / 1000000 << "ms"; } else { - auto schema = ChunkHelper::convert_schema(*_context.tablet_schema); + auto schema = ChunkHelper::convert_schema(_context.tablet_schema); for (const auto& segment : segments) { auto res = segment->new_iterator(schema, seg_options); @@ -926,7 +926,7 @@ Status HorizontalRowsetWriter::_final_merge() { if (st.is_end_of_file()) { break; } else if (st.ok()) { - ChunkHelper::padding_char_columns(char_field_indexes, schema, *_context.tablet_schema, chunk); + ChunkHelper::padding_char_columns(char_field_indexes, schema, _context.tablet_schema, chunk); total_rows += chunk->num_rows(); total_chunk++; if (auto st = add_chunk(*chunk); !st.ok()) { @@ -1152,7 +1152,7 @@ StatusOr> VerticalRowsetWriter::_create_segment_w std::lock_guard l(_lock); ASSIGN_OR_RETURN(auto wfile, _fs->new_writable_file(Rowset::segment_file_path(_context.rowset_path_prefix, _context.rowset_id, _num_segment))); - const auto* schema = _context.tablet_schema; + const auto schema = _context.tablet_schema; auto segment_writer = std::make_unique(std::move(wfile), _num_segment, schema, _writer_options); RETURN_IF_ERROR(segment_writer->init(column_indexes, is_key)); ++_num_segment; diff --git a/be/src/storage/rowset/rowset_writer_context.h b/be/src/storage/rowset/rowset_writer_context.h index 1b859cb11a53c..aa0fa2cf86cea 100644 --- a/be/src/storage/rowset/rowset_writer_context.h +++ b/be/src/storage/rowset/rowset_writer_context.h @@ -57,7 +57,7 @@ class RowsetWriterContext { std::string rowset_path_prefix; - const TabletSchema* tablet_schema = nullptr; + TabletSchemaCSPtr tablet_schema = nullptr; std::shared_ptr partial_update_tablet_schema = nullptr; std::vector referenced_column_ids; diff --git a/be/src/storage/rowset/segment.cpp b/be/src/storage/rowset/segment.cpp index d5e64efef9b7a..c72da21f3fd84 100644 --- a/be/src/storage/rowset/segment.cpp +++ b/be/src/storage/rowset/segment.cpp @@ -71,16 +71,6 @@ namespace starrocks { using strings::Substitute; -StatusOr> Segment::open(std::shared_ptr fs, const std::string& path, - uint32_t segment_id, const TabletSchema* tablet_schema, - size_t* footer_length_hint, - const FooterPointerPB* partial_rowset_footer) { - auto segment = std::make_shared(private_type(0), std::move(fs), path, segment_id, tablet_schema); - - RETURN_IF_ERROR(segment->_open(footer_length_hint, partial_rowset_footer, true)); - return std::move(segment); -} - StatusOr> Segment::open(std::shared_ptr fs, const std::string& path, uint32_t segment_id, std::shared_ptr tablet_schema, size_t* footer_length_hint, @@ -184,13 +174,7 @@ Status Segment::parse_segment_footer(RandomAccessFile* read_file, SegmentFooterP } Segment::Segment(const private_type&, std::shared_ptr fs, std::string path, uint32_t segment_id, - const TabletSchema* tablet_schema) - : _fs(std::move(fs)), _fname(std::move(path)), _tablet_schema(tablet_schema), _segment_id(segment_id) { - MEM_TRACKER_SAFE_CONSUME(GlobalEnv::GetInstance()->segment_metadata_mem_tracker(), _basic_info_mem_usage()); -} - -Segment::Segment(const private_type&, std::shared_ptr fs, std::string path, uint32_t segment_id, - std::shared_ptr tablet_schema) + TabletSchemaCSPtr tablet_schema) : _fs(std::move(fs)), _fname(std::move(path)), _tablet_schema(std::move(tablet_schema)), @@ -244,14 +228,17 @@ StatusOr Segment::_new_iterator(const Schema& schema, const Se // trying to prune the current segment by segment-level zone map for (const auto& pair : read_options.predicates_for_zone_map) { ColumnId column_id = pair.first; - if (_column_readers[column_id] == nullptr || !_column_readers[column_id]->has_zone_map()) { + const auto& tablet_column = read_options.tablet_schema ? read_options.tablet_schema->column(column_id) + : _tablet_schema->column(column_id); + auto column_unique_id = tablet_column.unique_id(); + if (_column_readers.count(column_unique_id) < 1 || !_column_readers.at(column_unique_id)->has_zone_map()) { continue; } - if (!_column_readers[column_id]->segment_zone_map_filter(pair.second)) { + if (!_column_readers.at(column_unique_id)->segment_zone_map_filter(pair.second)) { // skip segment zonemap filter when this segment has column files link to it. const TabletColumn& tablet_column = _tablet_schema->column(column_id); if (tablet_column.is_key() || _use_segment_zone_map_filter(read_options)) { - read_options.stats->segment_stats_filtered += _column_readers[column_id]->num_rows(); + read_options.stats->segment_stats_filtered += _column_readers.at(column_unique_id)->num_rows(); return Status::EndOfFile(strings::Substitute("End of file $0, empty iterator", _fname)); } else { break; @@ -268,8 +255,9 @@ StatusOr Segment::new_iterator(const Schema& schema, const Seg // If input schema is not match the actual meta, must convert the read_options according // to the actual format. And create an AdaptSegmentIterator to wrap if (_needs_chunk_adapter) { + auto& ref_tablet_schema = read_options.tablet_schema ? read_options.tablet_schema : _tablet_schema.schema(); std::unique_ptr adapter(new SegmentChunkIteratorAdapter( - *_tablet_schema, *_column_storage_types, schema, read_options.chunk_size)); + ref_tablet_schema, *_column_storage_types, schema, read_options.chunk_size)); RETURN_IF_ERROR(adapter->prepare(read_options)); auto result = _new_iterator(adapter->in_schema(), adapter->in_read_options()); @@ -332,6 +320,10 @@ bool Segment::has_loaded_index() const { return invoked(_load_index_once); } +bool Segment::is_valid_column(uint32_t column_unique_id) const { + return _column_readers.count(column_unique_id) > 0; +} + Status Segment::_create_column_readers(SegmentFooterPB* footer) { std::unordered_map column_id_to_footer_ordinal; for (uint32_t ordinal = 0, sz = footer->columns().size(); ordinal < sz; ++ordinal) { @@ -339,9 +331,8 @@ Status Segment::_create_column_readers(SegmentFooterPB* footer) { column_id_to_footer_ordinal.emplace(column_pb.unique_id(), ordinal); } - _column_readers.resize(_tablet_schema->columns().size()); for (uint32_t ordinal = 0, sz = _tablet_schema->num_columns(); ordinal < sz; ++ordinal) { - const auto& column = _tablet_schema->columns()[ordinal]; + const auto& column = _tablet_schema->column(ordinal); auto iter = column_id_to_footer_ordinal.find(column.unique_id()); if (iter == column_id_to_footer_ordinal.end()) { continue; @@ -351,7 +342,7 @@ Status Segment::_create_column_readers(SegmentFooterPB* footer) { if (!res.ok()) { return res.status(); } - _column_readers[ordinal] = std::move(res).value(); + _column_readers.emplace(column.unique_id(), std::move(res).value()); } return Status::OK(); } @@ -362,8 +353,9 @@ void Segment::_prepare_adapter_info() { std::vector types(num_columns); for (ColumnId cid = 0; cid < num_columns; ++cid) { LogicalType type; - if (_column_readers[cid] != nullptr) { - type = _column_readers[cid]->column_type(); + auto column_unique_id = _tablet_schema->column(cid).unique_id(); + if (_column_readers.count(column_unique_id) > 0) { + type = _column_readers.at(column_unique_id)->column_type(); } else { // when the default column is used, column reader will be null. // And the type will be same with the tablet schema. @@ -379,9 +371,12 @@ void Segment::_prepare_adapter_info() { } } -StatusOr> Segment::new_column_iterator(uint32_t cid, ColumnAccessPath* path) { - if (_column_readers[cid] == nullptr) { - const TabletColumn& tablet_column = _tablet_schema->column(cid); +StatusOr> Segment::new_column_iterator(uint32_t cid, ColumnAccessPath* path, + const TabletSchemaCSPtr& read_tablet_schema) { + auto const& current_tablet_schema = !read_tablet_schema ? _tablet_schema.schema() : read_tablet_schema; + const TabletColumn& tablet_column = current_tablet_schema->column(cid); + auto column_unique_id = tablet_column.unique_id(); + if ((_column_readers.count(column_unique_id) < 1)) { if (!tablet_column.has_default_value() && !tablet_column.is_nullable()) { return Status::InternalError( fmt::format("invalid nonexistent column({}) without default value.", tablet_column.name())); @@ -394,19 +389,21 @@ StatusOr> Segment::new_column_iterator(uint32_t RETURN_IF_ERROR(default_value_iter->init(iter_opts)); return default_value_iter; } - return _column_readers[cid]->new_iterator(path); + return _column_readers.at(column_unique_id)->new_iterator(path); } -Status Segment::new_bitmap_index_iterator(uint32_t cid, const IndexReadOptions& options, BitmapIndexIterator** iter) { - if (_column_readers[cid] != nullptr && _column_readers[cid]->has_bitmap_index()) { - return _column_readers[cid]->new_bitmap_index_iterator(options, iter); +Status Segment::new_bitmap_index_iterator(uint32_t cid, const IndexReadOptions& options, BitmapIndexIterator** iter, + const TabletSchemaCSPtr& tablet_schema) { + auto column_id = tablet_schema->column(cid).unique_id(); + if (_column_readers.count(column_id) > 0 && _column_readers.at(column_id)->has_bitmap_index()) { + return _column_readers.at(column_id)->new_bitmap_index_iterator(options, iter); } return Status::OK(); } StatusOr> Segment::new_dcg_segment(const DeltaColumnGroup& dcg, uint32_t idx) { return Segment::open(_fs, dcg.column_files(parent_name(_fname))[idx], 0, - TabletSchema::create_with_uid(*_tablet_schema, dcg.column_ids()[idx]), nullptr); + TabletSchema::create_with_uid(_tablet_schema.schema(), dcg.column_ids()[idx]), nullptr); } Status Segment::get_short_key_index(std::vector* sk_index_values) { diff --git a/be/src/storage/rowset/segment.h b/be/src/storage/rowset/segment.h index 283c890b7269a..a902b31f00c62 100644 --- a/be/src/storage/rowset/segment.h +++ b/be/src/storage/rowset/segment.h @@ -85,16 +85,9 @@ class Segment : public std::enable_shared_from_this { }; public: - // Does NOT take the ownership of |tablet_schema|. + // Like above but share the ownership of |unsafe_tablet_schema_ref|. static StatusOr> open(std::shared_ptr fs, const std::string& path, - uint32_t segment_id, const TabletSchema* tablet_schema, - size_t* footer_length_hint = nullptr, - const FooterPointerPB* partial_rowset_footer = nullptr); - - // Like above but share the ownership of |tablet_schema|. - static StatusOr> open(std::shared_ptr fs, const std::string& path, - uint32_t segment_id, - std::shared_ptr tablet_schema, + uint32_t segment_id, TabletSchemaCSPtr tablet_schema, size_t* footer_length_hint = nullptr, const FooterPointerPB* partial_rowset_footer = nullptr, bool skip_fill_local_cache = true); @@ -104,10 +97,7 @@ class Segment : public std::enable_shared_from_this { const FooterPointerPB* partial_rowset_footer); Segment(const private_type&, std::shared_ptr fs, std::string path, uint32_t segment_id, - const TabletSchema* tablet_schema); - - Segment(const private_type&, std::shared_ptr fs, std::string path, uint32_t segment_id, - std::shared_ptr tablet_schema); + TabletSchemaCSPtr tablet_schema); ~Segment(); @@ -119,10 +109,11 @@ class Segment : public std::enable_shared_from_this { uint64_t id() const { return _segment_id; } // TODO: remove this method, create `ColumnIterator` via `ColumnReader`. - StatusOr> new_column_iterator(uint32_t cid, ColumnAccessPath* path = nullptr); + StatusOr> new_column_iterator(uint32_t id, ColumnAccessPath* path = nullptr, + const TabletSchemaCSPtr& tablet_schema = nullptr); [[nodiscard]] Status new_bitmap_index_iterator(uint32_t cid, const IndexReadOptions& options, - BitmapIndexIterator** iter); + BitmapIndexIterator** iter, const TabletSchemaCSPtr& tablet_schema); size_t num_short_keys() const { return _tablet_schema->num_short_key_columns(); } @@ -152,12 +143,16 @@ class Segment : public std::enable_shared_from_this { size_t num_columns() const { return _column_readers.size(); } - const ColumnReader* column(size_t i) const { return _column_readers[i].get(); } + const ColumnReader* column(size_t i) const { + return _column_readers.at(_tablet_schema->column(i).unique_id()).get(); + } FileSystem* file_system() const { return _fs.get(); } const TabletSchema& tablet_schema() const { return *_tablet_schema; } + const TabletSchemaCSPtr tablet_schema_share_ptr() { return _tablet_schema.schema(); } + const std::string& file_name() const { return _fname; } uint32_t num_rows() const { return _num_rows; } @@ -171,6 +166,8 @@ class Segment : public std::enable_shared_from_this { int64_t mem_usage() { return _basic_info_mem_usage() + _short_key_index_mem_usage(); } + bool is_valid_column(uint32_t column_unique_id) const; + int64_t get_data_size() { auto res = _fs->get_file_size(_fname); if (res.ok()) { @@ -195,8 +192,10 @@ class Segment : public std::enable_shared_from_this { // Does not take the ownership of TabletSchema pointed by |raw_ptr|. explicit TabletSchemaWrapper(const TabletSchema* raw_ptr) : _schema(raw_ptr, DummyDeleter()) {} + explicit TabletSchemaWrapper(const TabletSchemaCSPtr* shared_ptr) : _schema(*shared_ptr) {} + // Shard the ownership of |ptr|. - explicit TabletSchemaWrapper(std::shared_ptr ptr) : _schema(std::move(ptr)) {} + explicit TabletSchemaWrapper(TabletSchemaCSPtr ptr) : _schema(std::move(ptr)) {} DISALLOW_COPY_AND_MOVE(TabletSchemaWrapper); @@ -204,8 +203,10 @@ class Segment : public std::enable_shared_from_this { const TabletSchema& operator*() const { return *_schema; } + const TabletSchemaCSPtr& schema() { return _schema; }; + private: - std::shared_ptr _schema; + TabletSchemaCSPtr _schema; }; Status _load_index(bool skip_fill_local_cache); @@ -244,7 +245,7 @@ class Segment : public std::enable_shared_from_this { // ColumnReader for each column in TabletSchema. If ColumnReader is nullptr, // This means that this segment has no data for that column, which may be added // after this segment is generated. - std::vector> _column_readers; + std::map> _column_readers; // used to guarantee that short key index will be loaded at most once in a thread-safe way OnceFlag _load_index_once; diff --git a/be/src/storage/rowset/segment_chunk_iterator_adapter.cpp b/be/src/storage/rowset/segment_chunk_iterator_adapter.cpp index bdfba962f5595..ef23c7185e029 100644 --- a/be/src/storage/rowset/segment_chunk_iterator_adapter.cpp +++ b/be/src/storage/rowset/segment_chunk_iterator_adapter.cpp @@ -14,15 +14,17 @@ #include "segment_chunk_iterator_adapter.h" +#include + #include "storage/chunk_helper.h" namespace starrocks { -SegmentChunkIteratorAdapter::SegmentChunkIteratorAdapter(const TabletSchema& tablet_schema, +SegmentChunkIteratorAdapter::SegmentChunkIteratorAdapter(const TabletSchemaCSPtr& tablet_schema, const std::vector& new_types, const Schema& out_schema, int chunk_size) : ChunkIterator(out_schema, chunk_size), - _tablet_schema(tablet_schema), + _tablet_schema(std::move(tablet_schema)), _new_types(new_types), _convert_time(0), _convert_timer(nullptr) {} diff --git a/be/src/storage/rowset/segment_chunk_iterator_adapter.h b/be/src/storage/rowset/segment_chunk_iterator_adapter.h index 75ab626231f49..411dbfb7c828a 100644 --- a/be/src/storage/rowset/segment_chunk_iterator_adapter.h +++ b/be/src/storage/rowset/segment_chunk_iterator_adapter.h @@ -31,8 +31,9 @@ namespace starrocks { class SegmentChunkIteratorAdapter final : public ChunkIterator { public: // |schema| is the output fields. - explicit SegmentChunkIteratorAdapter(const TabletSchema& tablet_schema, const std::vector& new_types, - const Schema& out_schema, int chunk_size); + explicit SegmentChunkIteratorAdapter(const TabletSchemaCSPtr& tablet_schema, + const std::vector& new_types, const Schema& out_schema, + int chunk_size); ~SegmentChunkIteratorAdapter() override = default; @@ -60,7 +61,7 @@ class SegmentChunkIteratorAdapter final : public ChunkIterator { Status do_get_next(Chunk* chunk) override; Status do_get_next(Chunk* chunk, std::vector* rowid) override; - const TabletSchema& _tablet_schema; + const TabletSchemaCSPtr _tablet_schema; const std::vector& _new_types; Schema _in_schema; diff --git a/be/src/storage/rowset/segment_iterator.cpp b/be/src/storage/rowset/segment_iterator.cpp index d67477a5d06e5..86e16b09a15d2 100644 --- a/be/src/storage/rowset/segment_iterator.cpp +++ b/be/src/storage/rowset/segment_iterator.cpp @@ -459,8 +459,8 @@ StatusOr> SegmentIterator::_get_dcg_segment(uint32_t uc */ DCHECK(_dcg_segments[column_file]->num_columns() <= dcg->column_ids()[idx.first].size()); if (_dcg_segments[column_file]->num_columns() < dcg->column_ids()[idx.first].size()) { - const auto& new_schema = - TabletSchema::create_with_uid(_segment->tablet_schema(), dcg->column_ids()[idx.first]); + const auto& new_schema = TabletSchema::create_with_uid(_segment->tablet_schema_share_ptr(), + dcg->column_ids()[idx.first]); *col_index = INT32_MIN; for (int i = 0; i < new_schema->columns().size(); ++i) { @@ -539,7 +539,7 @@ Status SegmentIterator::_init_column_iterator_by_cid(const ColumnId cid, const C ASSIGN_OR_RETURN(auto col_iter, _new_dcg_column_iterator((uint32_t)ucid, &dcg_filename, access_path)); if (col_iter == nullptr) { // not found in delta column group, create normal column iterator - ASSIGN_OR_RETURN(_column_iterators[cid], _segment->new_column_iterator(cid, access_path)); + ASSIGN_OR_RETURN(_column_iterators[cid], _segment->new_column_iterator(cid, access_path, _opts.tablet_schema)); ASSIGN_OR_RETURN(auto rfile, _opts.fs->new_random_access_file(opts, _segment->file_name())); iter_opts.read_file = rfile.get(); _column_files[cid] = std::move(rfile); @@ -1590,7 +1590,8 @@ Status SegmentIterator::_init_bitmap_index_iterators() { opts.read_file = _column_files[cid].get(); opts.stats = _opts.stats; - RETURN_IF_ERROR(segment_ptr->new_bitmap_index_iterator(col_index, opts, &_bitmap_index_iterators[cid])); + RETURN_IF_ERROR(segment_ptr->new_bitmap_index_iterator(col_index, opts, &_bitmap_index_iterators[cid], + _opts.tablet_schema)); _has_bitmap_index |= (_bitmap_index_iterators[cid] != nullptr); } } diff --git a/be/src/storage/rowset/segment_options.cpp b/be/src/storage/rowset/segment_options.cpp index 067b277f2ea6c..7553a18868847 100644 --- a/be/src/storage/rowset/segment_options.cpp +++ b/be/src/storage/rowset/segment_options.cpp @@ -75,7 +75,7 @@ std::string SegmentReadOptions::debug_string() const { ss << "]}"; } ss << "],delete_predicates={"; - ss << "},tablet_schema={"; + ss << "},unsafe_tablet_schema_ref={"; ss << "},use_page_cache=" << use_page_cache; return ss.str(); } diff --git a/be/src/storage/rowset/segment_options.h b/be/src/storage/rowset/segment_options.h index ef1ab6b8b5619..9001dd06333b0 100644 --- a/be/src/storage/rowset/segment_options.h +++ b/be/src/storage/rowset/segment_options.h @@ -25,6 +25,7 @@ #include "storage/disjunctive_predicates.h" #include "storage/olap_runtime_range_pruner.h" #include "storage/seek_range.h" +#include "storage/tablet_schema.h" namespace starrocks { class Condition; @@ -92,6 +93,8 @@ class SegmentReadOptions { RowsetId rowsetid; + TabletSchemaCSPtr tablet_schema = nullptr; + public: Status convert_to(SegmentReadOptions* dst, const std::vector& new_types, ObjectPool* obj_pool) const; diff --git a/be/src/storage/rowset/segment_rewriter.cpp b/be/src/storage/rowset/segment_rewriter.cpp index 8bf19dd591878..60fb4faa61488 100644 --- a/be/src/storage/rowset/segment_rewriter.cpp +++ b/be/src/storage/rowset/segment_rewriter.cpp @@ -20,9 +20,10 @@ namespace starrocks { SegmentRewriter::SegmentRewriter() = default; -Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& dest_path, const TabletSchema& tschema, - std::vector& column_ids, std::vector>& columns, - uint32_t segment_id, const FooterPointerPB& partial_rowset_footer) { +Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& dest_path, + const std::shared_ptr& tschema, std::vector& column_ids, + std::vector>& columns, uint32_t segment_id, + const FooterPointerPB& partial_rowset_footer) { ASSIGN_OR_RETURN(auto fs, FileSystem::CreateSharedFromString(dest_path)); WritableFileOptions wopts{.sync_on_close = true, .mode = FileSystem::CREATE_OR_OPEN_WITH_TRUNCATE}; ASSIGN_OR_RETURN(auto wfile, fs->new_writable_file(wopts, dest_path)); @@ -49,7 +50,7 @@ Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& } SegmentWriterOptions opts; - SegmentWriter writer(std::move(wfile), segment_id, &tschema, opts); + SegmentWriter writer(std::move(wfile), segment_id, tschema, opts); RETURN_IF_ERROR(writer.init(column_ids, false, &footer)); auto schema = ChunkHelper::convert_schema(tschema, column_ids); @@ -69,7 +70,8 @@ Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& // This function is used when the auto-increment column is not specified in partial update. // In this function, we use the segment iterator to read the old data, replace the old auto // increment column, and rewrite the full segment file through SegmentWriter. -Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& dest_path, const TabletSchema& tschema, +Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& dest_path, + const TabletSchemaCSPtr& tschema, AutoIncrementPartialUpdateState& auto_increment_partial_update_state, std::vector& column_ids, std::vector>* columns) { if (column_ids.size() == 0) { @@ -79,7 +81,7 @@ Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& ASSIGN_OR_RETURN(auto fs, FileSystem::CreateSharedFromString(dest_path)); uint32_t auto_increment_column_id = 0; - for (const auto& col : tschema.columns()) { + for (const auto& col : tschema->columns()) { if (col.is_auto_increment()) { break; } @@ -94,7 +96,7 @@ Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& std::vector src_column_ids; std::set update_columns_set(column_ids.begin(), column_ids.end()); - for (auto i = 0; i < tschema.num_columns(); ++i) { + for (auto i = 0; i < tschema->num_columns(); ++i) { if (i != auto_increment_column_id && update_columns_set.find(i) == update_columns_set.end()) { src_column_ids.emplace_back(i); } @@ -122,14 +124,14 @@ Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& WritableFileOptions wopts{.sync_on_close = true, .mode = FileSystem::CREATE_OR_OPEN_WITH_TRUNCATE}; ASSIGN_OR_RETURN(auto wfile, fs->new_writable_file(wopts, dest_path)); - std::vector full_column_ids(tschema.num_columns()); + std::vector full_column_ids(tschema->num_columns()); std::iota(full_column_ids.begin(), full_column_ids.end(), 0); auto schema = ChunkHelper::convert_schema(tschema, full_column_ids); auto chunk = ChunkHelper::new_chunk(schema, full_column_ids.size()); size_t update_columns_index = 0; size_t read_columns_index = 0; - for (int i = 0; i < tschema.num_columns(); ++i) { + for (int i = 0; i < tschema->num_columns(); ++i) { if (i == auto_increment_column_id) { chunk->get_column_by_index(i).reset(auto_increment_partial_update_state.write_column.release()); } else if (update_columns_set.find(i) != update_columns_set.end()) { @@ -142,7 +144,7 @@ Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& } SegmentWriterOptions opts; - SegmentWriter writer(std::move(wfile), segment_id, &tschema, opts); + SegmentWriter writer(std::move(wfile), segment_id, tschema, opts); RETURN_IF_ERROR(writer.init(full_column_ids, true)); uint64_t index_size = 0; @@ -157,7 +159,8 @@ Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& // This function is used when the auto-increment column is not specified in partial update. // In this function, we use the segment iterator to read the old data, replace the old auto // increment column, and rewrite the full segment file through SegmentWriter. -Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& dest_path, const TabletSchema& tschema, +Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& dest_path, + const TabletSchemaCSPtr& tschema, starrocks::lake::AutoIncrementPartialUpdateState& auto_increment_partial_update_state, std::vector& column_ids, std::vector>* columns, const starrocks::lake::TxnLogPB_OpWrite& op_write, starrocks::lake::Tablet* tablet) { @@ -168,7 +171,7 @@ Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& ASSIGN_OR_RETURN(auto fs, FileSystem::CreateSharedFromString(dest_path)); uint32_t auto_increment_column_id = 0; - for (const auto& col : tschema.columns()) { + for (const auto& col : tschema->columns()) { if (col.is_auto_increment()) { break; } @@ -179,7 +182,7 @@ Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& std::vector src_column_ids; std::set update_columns_set(column_ids.begin(), column_ids.end()); - for (auto i = 0; i < tschema.num_columns(); ++i) { + for (auto i = 0; i < tschema->num_columns(); ++i) { if (i != auto_increment_column_id && update_columns_set.find(i) == update_columns_set.end()) { src_column_ids.emplace_back(i); } @@ -213,14 +216,14 @@ Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& WritableFileOptions wopts{.sync_on_close = true, .mode = FileSystem::CREATE_OR_OPEN_WITH_TRUNCATE}; ASSIGN_OR_RETURN(auto wfile, fs->new_writable_file(wopts, dest_path)); - std::vector full_column_ids(tschema.num_columns()); + std::vector full_column_ids(tschema->num_columns()); std::iota(full_column_ids.begin(), full_column_ids.end(), 0); auto schema = ChunkHelper::convert_schema(tschema, full_column_ids); auto chunk = ChunkHelper::new_chunk(schema, full_column_ids.size()); size_t update_columns_index = 0; size_t read_columns_index = 0; - for (int i = 0; i < tschema.num_columns(); ++i) { + for (int i = 0; i < tschema->num_columns(); ++i) { if (i == auto_increment_column_id) { chunk->get_column_by_index(i).reset(auto_increment_partial_update_state.write_column.release()); } else if (update_columns_set.find(i) != update_columns_set.end()) { @@ -233,7 +236,7 @@ Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& } SegmentWriterOptions opts; - SegmentWriter writer(std::move(wfile), segment_id, &tschema, opts); + SegmentWriter writer(std::move(wfile), segment_id, tschema, opts); RETURN_IF_ERROR(writer.init(full_column_ids, true)); uint64_t index_size = 0; @@ -245,7 +248,7 @@ Status SegmentRewriter::rewrite(const std::string& src_path, const std::string& return Status::OK(); } -Status SegmentRewriter::rewrite(const std::string& src_path, const TabletSchema& tschema, +Status SegmentRewriter::rewrite(const std::string& src_path, const TabletSchemaCSPtr& tschema, std::vector& column_ids, std::vector>& columns, uint32_t segment_id, const FooterPointerPB& partial_rowset_footer) { ASSIGN_OR_RETURN(auto fs, FileSystem::CreateSharedFromString(src_path)); @@ -261,7 +264,7 @@ Status SegmentRewriter::rewrite(const std::string& src_path, const TabletSchema& ASSIGN_OR_RETURN(auto wfile, fs->new_writable_file(fopts, src_path)); SegmentWriterOptions opts; - SegmentWriter writer(std::move(wfile), segment_id, &tschema, opts); + SegmentWriter writer(std::move(wfile), segment_id, tschema, opts); RETURN_IF_ERROR(writer.init(column_ids, false, &footer)); auto schema = ChunkHelper::convert_schema(tschema, column_ids); diff --git a/be/src/storage/rowset/segment_rewriter.h b/be/src/storage/rowset/segment_rewriter.h index f117858444c10..90af5c5d21cb1 100644 --- a/be/src/storage/rowset/segment_rewriter.h +++ b/be/src/storage/rowset/segment_rewriter.h @@ -26,17 +26,18 @@ class SegmentRewriter { // read from src, write to dest // this function will read data from src_file and write to dest file first // then append write_column to dest file - static Status rewrite(const std::string& src, const std::string& dest, const TabletSchema& tschema, - std::vector& column_ids, std::vector>& columns, - uint32_t segment_id, const FooterPointerPB& partial_rowseet_footer); + static Status rewrite(const std::string& src, const std::string& dest, + const std::shared_ptr& tschema, std::vector& column_ids, + std::vector>& columns, uint32_t segment_id, + const FooterPointerPB& partial_rowseet_footer); // this funciton will append write_column to src_file and rebuild segment footer - static Status rewrite(const std::string& src, const TabletSchema& tschema, std::vector& column_ids, + static Status rewrite(const std::string& src, const TabletSchemaCSPtr& tschema, std::vector& column_ids, std::vector>& columns, uint32_t segment_id, const FooterPointerPB& partial_rowseet_footer); - static Status rewrite(const std::string& src_path, const std::string& dest_path, const TabletSchema& tschema, + static Status rewrite(const std::string& src_path, const std::string& dest_path, const TabletSchemaCSPtr& tschema, AutoIncrementPartialUpdateState& auto_increment_partial_update_state, std::vector& column_ids, std::vector>* columns); - static Status rewrite(const std::string& src_path, const std::string& dest_path, const TabletSchema& tschema, + static Status rewrite(const std::string& src_path, const std::string& dest_path, const TabletSchemaCSPtr& tschema, starrocks::lake::AutoIncrementPartialUpdateState& auto_increment_partial_update_state, std::vector& column_ids, std::vector>* columns, const starrocks::lake::TxnLogPB_OpWrite& op_write, starrocks::lake::Tablet* tablet); diff --git a/be/src/storage/rowset/segment_writer.cpp b/be/src/storage/rowset/segment_writer.cpp index ed571ac221d11..8135ae8382304 100644 --- a/be/src/storage/rowset/segment_writer.cpp +++ b/be/src/storage/rowset/segment_writer.cpp @@ -57,9 +57,12 @@ namespace starrocks { const char* const k_segment_magic = "D0R1"; const uint32_t k_segment_magic_length = 4; -SegmentWriter::SegmentWriter(std::unique_ptr wfile, uint32_t segment_id, - const TabletSchema* tablet_schema, SegmentWriterOptions opts) - : _segment_id(segment_id), _tablet_schema(tablet_schema), _opts(std::move(opts)), _wfile(std::move(wfile)) { +SegmentWriter::SegmentWriter(std::unique_ptr wfile, uint32_t segment_id, TabletSchemaCSPtr tablet_schema, + SegmentWriterOptions opts) + : _segment_id(segment_id), + _tablet_schema(std::move(tablet_schema)), + _opts(std::move(opts)), + _wfile(std::move(wfile)) { CHECK_NOTNULL(_wfile.get()); } diff --git a/be/src/storage/rowset/segment_writer.h b/be/src/storage/rowset/segment_writer.h index 8129451b26719..86767b3fa3835 100644 --- a/be/src/storage/rowset/segment_writer.h +++ b/be/src/storage/rowset/segment_writer.h @@ -43,6 +43,7 @@ #include "gen_cpp/segment.pb.h" #include "gutil/macros.h" #include "runtime/global_dict/types.h" +#include "storage/tablet_schema.h" namespace starrocks { @@ -93,7 +94,7 @@ struct SegmentWriterOptions { // class SegmentWriter { public: - SegmentWriter(std::unique_ptr block, uint32_t segment_id, const TabletSchema* tablet_schema, + SegmentWriter(std::unique_ptr block, uint32_t segment_id, TabletSchemaCSPtr tablet_schema, SegmentWriterOptions opts); ~SegmentWriter(); @@ -138,7 +139,7 @@ class SegmentWriter { void _init_column_meta(ColumnMetaPB* meta, uint32_t column_id, const TabletColumn& column); uint32_t _segment_id; - const TabletSchema* _tablet_schema; + TabletSchemaCSPtr _tablet_schema; SegmentWriterOptions _opts; std::unique_ptr _wfile; diff --git a/be/src/storage/rowset_column_update_state.cpp b/be/src/storage/rowset_column_update_state.cpp index 24ee7402b5f5f..a0c4e999a1e16 100644 --- a/be/src/storage/rowset_column_update_state.cpp +++ b/be/src/storage/rowset_column_update_state.cpp @@ -102,7 +102,7 @@ Status RowsetColumnUpdateState::_load_upserts(Rowset* rowset, uint32_t idx) { OlapReaderStatistics stats; auto& schema = rowset->schema(); vector pk_columns; - for (size_t i = 0; i < schema.num_key_columns(); i++) { + for (size_t i = 0; i < schema->num_key_columns(); i++) { pk_columns.push_back((uint32_t)i); } std::vector update_columns; @@ -110,7 +110,7 @@ Status RowsetColumnUpdateState::_load_upserts(Rowset* rowset, uint32_t idx) { const auto& txn_meta = rowset->rowset_meta()->get_meta_pb().txn_meta(); for (uint32_t cid : txn_meta.partial_update_column_ids()) { update_columns_with_keys.push_back(cid); - if (cid >= schema.num_key_columns()) { + if (cid >= schema->num_key_columns()) { update_columns.push_back(cid); } } @@ -304,7 +304,7 @@ static StatusOr read_from_source_segment(Rowset* rowset, const Schema& OlapReaderStatistics* stats, int64_t version, RowsetSegmentId rowset_seg_id, const std::string& path) { ASSIGN_OR_RETURN(auto fs, FileSystem::CreateSharedFromString(rowset->rowset_path())); - auto segment = Segment::open(fs, path, rowset_seg_id.segment_id, &rowset->schema()); + auto segment = Segment::open(fs, path, rowset_seg_id.segment_id, rowset->schema()); if (!segment.ok()) { LOG(WARNING) << "Fail to open " << path << ": " << segment.status(); return segment.status(); @@ -351,7 +351,7 @@ StatusOr> RowsetColumnUpdateState::_prepare_delta ASSIGN_OR_RETURN(auto wfile, fs->new_writable_file(opts, path)); SegmentWriterOptions writer_options; auto segment_writer = - std::make_unique(std::move(wfile), rowsetid_segid.segment_id, tschema.get(), writer_options); + std::make_unique(std::move(wfile), rowsetid_segid.segment_id, tschema, writer_options); RETURN_IF_ERROR(segment_writer->init(false)); return std::move(segment_writer); } @@ -427,26 +427,26 @@ Status RowsetColumnUpdateState::_read_chunk_from_update(const RowidsToUpdateRowi // this function build segment writer for segment files StatusOr> RowsetColumnUpdateState::_prepare_segment_writer( - Rowset* rowset, const TabletSchema& tablet_schema, int segment_id) { + Rowset* rowset, const TabletSchemaCSPtr& tablet_schema, int segment_id) { ASSIGN_OR_RETURN(auto fs, FileSystem::CreateSharedFromString(rowset->rowset_path())); const std::string path = Rowset::segment_file_path(rowset->rowset_path(), rowset->rowset_id(), segment_id); (void)fs->delete_file(path); // delete .dat if already exist WritableFileOptions opts{.sync_on_close = true}; ASSIGN_OR_RETURN(auto wfile, fs->new_writable_file(opts, path)); SegmentWriterOptions writer_options; - auto segment_writer = std::make_unique(std::move(wfile), segment_id, &tablet_schema, writer_options); + auto segment_writer = std::make_unique(std::move(wfile), segment_id, tablet_schema, writer_options); RETURN_IF_ERROR(segment_writer->init()); return std::move(segment_writer); } static std::pair, std::vector> get_read_update_columns_ids( - const RowsetTxnMetaPB& txn_meta, const TabletSchema& tablet_schema) { + const RowsetTxnMetaPB& txn_meta, const TabletSchemaCSPtr& tablet_schema) { std::vector update_column_ids(txn_meta.partial_update_column_ids().begin(), txn_meta.partial_update_column_ids().end()); std::set update_columns_set(update_column_ids.begin(), update_column_ids.end()); std::vector read_column_ids; - for (uint32_t i = 0; i < tablet_schema.num_columns(); i++) { + for (uint32_t i = 0; i < tablet_schema->num_columns(); i++) { if (update_columns_set.find(i) == update_columns_set.end()) { read_column_ids.push_back(i); } @@ -455,11 +455,11 @@ static std::pair, std::vector> get_read_update_c return {read_column_ids, update_column_ids}; } -Status RowsetColumnUpdateState::_fill_default_columns(const TabletSchema& tablet_schema, +Status RowsetColumnUpdateState::_fill_default_columns(const TabletSchemaCSPtr& tablet_schema, const std::vector& column_ids, const int64_t row_cnt, vector>* columns) { for (auto i = 0; i < column_ids.size(); ++i) { - const TabletColumn& tablet_column = tablet_schema.column(column_ids[i]); + const TabletColumn& tablet_column = tablet_schema->column(column_ids[i]); if (tablet_column.has_default_value()) { const TypeInfoPtr& type_info = get_type_info(tablet_column); std::unique_ptr default_value_iter = @@ -476,7 +476,7 @@ Status RowsetColumnUpdateState::_fill_default_columns(const TabletSchema& tablet return Status::OK(); } -Status RowsetColumnUpdateState::_update_primary_index(const TabletSchema& tablet_schema, Tablet* tablet, +Status RowsetColumnUpdateState::_update_primary_index(const TabletSchemaCSPtr& tablet_schema, Tablet* tablet, const EditVersion& edit_version, uint32_t rowset_id, std::map& segid_to_chunk, int64_t insert_row_cnt, PersistentIndexMetaPB& index_meta, @@ -484,7 +484,7 @@ Status RowsetColumnUpdateState::_update_primary_index(const TabletSchema& tablet PrimaryIndex& index) { // 1. build pk column vector pk_column_ids; - for (size_t i = 0; i < tablet_schema.num_key_columns(); i++) { + for (size_t i = 0; i < tablet_schema->num_key_columns(); i++) { pk_column_ids.push_back((uint32_t)i); } Schema pkey_schema = ChunkHelper::convert_schema(tablet_schema, pk_column_ids); @@ -525,13 +525,13 @@ Status RowsetColumnUpdateState::_update_rowset_meta(const RowsetSegmentStat& sta return Status::OK(); } -static void padding_char_columns(const Schema& schema, const TabletSchema& tschema, Chunk* chunk) { +static void padding_char_columns(const Schema& schema, const TabletSchemaCSPtr& tschema, Chunk* chunk) { auto char_field_indexes = ChunkHelper::get_char_field_indexes(schema); ChunkHelper::padding_char_columns(char_field_indexes, schema, tschema, chunk); } // handle new rows, generate segment files and update primary index -Status RowsetColumnUpdateState::_insert_new_rows(const TabletSchema& tablet_schema, Tablet* tablet, +Status RowsetColumnUpdateState::_insert_new_rows(const TabletSchemaCSPtr& tablet_schema, Tablet* tablet, const EditVersion& edit_version, Rowset* rowset, uint32_t rowset_id, PersistentIndexMetaPB& index_meta, vector>& delvecs, @@ -607,13 +607,13 @@ Status RowsetColumnUpdateState::finalize(Tablet* tablet, Rowset* rowset, uint32_ std::vector unique_update_column_ids; const auto& tschema = rowset->schema(); for (int32_t cid : txn_meta.partial_update_column_ids()) { - if (cid >= tschema.num_key_columns()) { + if (cid >= tschema->num_key_columns()) { update_column_ids.push_back(cid); update_column_uids.push_back((uint32_t)cid); } } for (uint32_t cid : txn_meta.partial_update_column_unique_ids()) { - auto& column = tschema.column(cid); + auto& column = tschema->column(cid); if (!column.is_key()) { unique_update_column_ids.push_back(cid); } @@ -679,7 +679,7 @@ Status RowsetColumnUpdateState::finalize(Tablet* tablet, Rowset* rowset, uint32_ uint64_t segment_file_size = 0; uint64_t index_size = 0; uint64_t footer_position = 0; - padding_char_columns(partial_schema, *partial_tschema, source_chunk_ptr.get()); + padding_char_columns(partial_schema, partial_tschema, source_chunk_ptr.get()); RETURN_IF_ERROR(delta_column_group_writer[each.first]->append_chunk(*source_chunk_ptr)); RETURN_IF_ERROR( delta_column_group_writer[each.first]->finalize(&segment_file_size, &index_size, &footer_position)); diff --git a/be/src/storage/rowset_column_update_state.h b/be/src/storage/rowset_column_update_state.h index 992004f5adf22..2de8b7146ba79 100644 --- a/be/src/storage/rowset_column_update_state.h +++ b/be/src/storage/rowset_column_update_state.h @@ -164,18 +164,20 @@ class RowsetColumnUpdateState { void _check_if_preload_column_mode_update_data(Rowset* rowset, MemTracker* update_mem_tracker); - StatusOr> _prepare_segment_writer(Rowset* rowset, const TabletSchema& tablet_schema, + StatusOr> _prepare_segment_writer(Rowset* rowset, + const TabletSchemaCSPtr& tablet_schema, int segment_id); - Status _fill_default_columns(const TabletSchema& tablet_schema, const std::vector& column_ids, + Status _fill_default_columns(const TabletSchemaCSPtr& tablet_schema, const std::vector& column_ids, const int64_t row_cnt, vector>* columns); - Status _update_primary_index(const TabletSchema& tablet_schema, Tablet* tablet, const EditVersion& edit_version, - uint32_t rowset_id, std::map& segid_to_chunk, - int64_t insert_row_cnt, PersistentIndexMetaPB& index_meta, - vector>& delvecs, PrimaryIndex& index); + Status _update_primary_index(const TabletSchemaCSPtr& tablet_schema, Tablet* tablet, + const EditVersion& edit_version, uint32_t rowset_id, + std::map& segid_to_chunk, int64_t insert_row_cnt, + PersistentIndexMetaPB& index_meta, vector>& delvecs, + PrimaryIndex& index); Status _update_rowset_meta(const RowsetSegmentStat& stat, Rowset* rowset); - Status _insert_new_rows(const TabletSchema& tablet_schema, Tablet* tablet, const EditVersion& edit_version, + Status _insert_new_rows(const TabletSchemaCSPtr& tablet_schema, Tablet* tablet, const EditVersion& edit_version, Rowset* rowset, uint32_t rowset_id, PersistentIndexMetaPB& index_meta, vector>& delvecs, PrimaryIndex& index); diff --git a/be/src/storage/rowset_merger.cpp b/be/src/storage/rowset_merger.cpp index 38071ac2d7ea2..1d1f3e8d2cfd8 100644 --- a/be/src/storage/rowset_merger.cpp +++ b/be/src/storage/rowset_merger.cpp @@ -326,7 +326,7 @@ class RowsetMergerImpl : public RowsetMerger { std::unique_ptr> source_masks; if (mask_buffer) { source_masks = std::make_unique>(); - column_indexes = tablet.tablet_schema().sort_key_idxes(); + column_indexes = tablet.tablet_schema()->sort_key_idxes(); } auto chunk = ChunkHelper::new_chunk(schema, _chunk_size); @@ -421,8 +421,9 @@ class RowsetMergerImpl : public RowsetMerger { rowsets_mask_buffer.emplace_back(std::move(rowset_mask_buffer)); } { - Schema schema = tablet.tablet_schema().sort_key_idxes().empty() - ? ChunkHelper::convert_schema(tablet.tablet_schema(), column_groups[0]) + auto tablet_schema_ptr = tablet.tablet_schema(); + Schema schema = tablet_schema_ptr->sort_key_idxes().empty() + ? ChunkHelper::convert_schema(tablet_schema_ptr, column_groups[0]) : ChunkHelper::get_sort_key_schema(tablet.tablet_schema()); RETURN_IF_ERROR(_do_merge_horizontally(tablet, version, schema, rowsets, writer, cfg, total_input_size, total_rows, total_chunk, stats, mask_buffer.get(), @@ -431,6 +432,7 @@ class RowsetMergerImpl : public RowsetMerger { // merge non key columns auto source_masks = std::make_unique>(); + auto tablet_schema_ptr = tablet.tablet_schema(); for (size_t i = 1; i < column_groups.size(); ++i) { // read mask buffer from the beginning mask_buffer->flip_to_read(); @@ -440,7 +442,7 @@ class RowsetMergerImpl : public RowsetMerger { vector iterators; iterators.reserve(rowsets.size()); OlapReaderStatistics non_key_stats; - Schema schema = ChunkHelper::convert_schema(tablet.tablet_schema(), column_groups[i]); + Schema schema = ChunkHelper::convert_schema(tablet_schema_ptr, column_groups[i]); for (size_t j = 0; j < rowsets.size(); j++) { const auto& rowset = rowsets[j]; rowsets_mask_buffer[j]->flip_to_read(); @@ -485,7 +487,7 @@ class RowsetMergerImpl : public RowsetMerger { } } - ChunkHelper::padding_char_columns(char_field_indexes, schema, tablet.tablet_schema(), chunk.get()); + ChunkHelper::padding_char_columns(char_field_indexes, schema, tablet_schema_ptr, chunk.get()); if (auto st = writer->add_columns(*chunk, column_groups[i], false); !st.ok()) { LOG(WARNING) << "writer add_columns error, tablet=" << tablet.tablet_id() << ", err=" << st; @@ -528,12 +530,15 @@ class RowsetMergerImpl : public RowsetMerger { }; Status compaction_merge_rowsets(Tablet& tablet, int64_t version, const vector& rowsets, - RowsetWriter* writer, const MergeConfig& cfg) { - Schema schema = [&tablet]() { - if (tablet.tablet_schema().sort_key_idxes().empty()) { - return ChunkHelper::get_sort_key_schema_by_primary_key(tablet.tablet_schema()); + RowsetWriter* writer, const MergeConfig& cfg, + const starrocks::TabletSchemaCSPtr& cur_tablet_schema) { + Schema schema = [&cur_tablet_schema, &tablet]() { + const starrocks::TabletSchemaCSPtr final_tablet_schema = + cur_tablet_schema == nullptr ? tablet.tablet_schema() : cur_tablet_schema; + if (final_tablet_schema->sort_key_idxes().empty()) { + return ChunkHelper::get_sort_key_schema_by_primary_key(final_tablet_schema); } else { - return ChunkHelper::convert_schema(tablet.tablet_schema()); + return ChunkHelper::convert_schema(final_tablet_schema); } }(); std::unique_ptr merger; diff --git a/be/src/storage/rowset_merger.h b/be/src/storage/rowset_merger.h index e3da91026076b..3e745bca82856 100644 --- a/be/src/storage/rowset_merger.h +++ b/be/src/storage/rowset_merger.h @@ -28,6 +28,7 @@ struct MergeConfig { // heap based rowset merger used for updatable tablet's compaction Status compaction_merge_rowsets(Tablet& tablet, int64_t version, const vector& rowsets, - RowsetWriter* writer, const MergeConfig& cfg); + RowsetWriter* writer, const MergeConfig& cfg, + const starrocks::TabletSchemaCSPtr& cur_tablet_schema = nullptr); } // namespace starrocks diff --git a/be/src/storage/rowset_update_state.cpp b/be/src/storage/rowset_update_state.cpp index 7c3a7969554f9..e15297405fb66 100644 --- a/be/src/storage/rowset_update_state.cpp +++ b/be/src/storage/rowset_update_state.cpp @@ -96,9 +96,9 @@ Status RowsetUpdateState::_load_upserts(Rowset* rowset, uint32_t idx, Column* pk } OlapReaderStatistics stats; - auto& schema = rowset->schema(); + const auto& schema = rowset->schema(); vector pk_columns; - for (size_t i = 0; i < schema.num_key_columns(); i++) { + for (size_t i = 0; i < schema->num_key_columns(); i++) { pk_columns.push_back((uint32_t)i); } Schema pkey_schema = ChunkHelper::convert_schema(schema, pk_columns); @@ -149,9 +149,9 @@ Status RowsetUpdateState::_do_load(Tablet* tablet, Rowset* rowset) { auto span = Tracer::Instance().start_trace_txn_tablet("rowset_update_state_load", rowset->txn_id(), tablet->tablet_id()); _tablet_id = tablet->tablet_id(); - auto& schema = rowset->schema(); + const auto& schema = rowset->schema(); vector pk_columns; - for (size_t i = 0; i < schema.num_key_columns(); i++) { + for (size_t i = 0; i < schema->num_key_columns(); i++) { pk_columns.push_back((uint32_t)i); } Schema pkey_schema = ChunkHelper::convert_schema(schema, pk_columns); @@ -183,9 +183,9 @@ Status RowsetUpdateState::_do_load(Tablet* tablet, Rowset* rowset) { } Status RowsetUpdateState::load_deletes(Rowset* rowset, uint32_t idx) { - auto& schema = rowset->schema(); + const auto& schema = rowset->schema(); vector pk_columns; - for (size_t i = 0; i < schema.num_key_columns(); i++) { + for (size_t i = 0; i < schema->num_key_columns(); i++) { pk_columns.push_back((uint32_t)i); } Schema pkey_schema = ChunkHelper::convert_schema(schema, pk_columns); @@ -197,9 +197,9 @@ Status RowsetUpdateState::load_deletes(Rowset* rowset, uint32_t idx) { } Status RowsetUpdateState::load_upserts(Rowset* rowset, uint32_t upsert_id) { - auto& schema = rowset->schema(); + const auto& schema = rowset->schema(); vector pk_columns; - for (size_t i = 0; i < schema.num_key_columns(); i++) { + for (size_t i = 0; i < schema->num_key_columns(); i++) { pk_columns.push_back((uint32_t)i); } Schema pkey_schema = ChunkHelper::convert_schema(schema, pk_columns); @@ -318,14 +318,14 @@ Status RowsetUpdateState::_prepare_partial_update_states(Tablet* tablet, Rowset* int64_t t_start = MonotonicMillis(); const auto& txn_meta = rowset->rowset_meta()->get_meta_pb().txn_meta(); - const auto& tablet_schema = tablet->tablet_schema(); + const auto tablet_schema = tablet->tablet_schema(); std::vector update_column_ids(txn_meta.partial_update_column_ids().begin(), txn_meta.partial_update_column_ids().end()); std::set update_columns_set(update_column_ids.begin(), update_column_ids.end()); std::vector read_column_ids; - for (uint32_t i = 0; i < tablet_schema.num_columns(); i++) { + for (uint32_t i = 0; i < tablet_schema->num_columns(); i++) { if (update_columns_set.find(i) == update_columns_set.end()) { read_column_ids.push_back(i); } @@ -401,9 +401,9 @@ Status RowsetUpdateState::_prepare_auto_increment_partial_update_states(Tablet* schema = TabletSchema::create(tablet->tablet_schema(), update_column_ids); } - _auto_increment_partial_update_states[idx].init( - rowset, schema != nullptr ? schema.get() : const_cast(&tablet->tablet_schema()), - rowset_meta_pb.txn_meta().auto_increment_partial_update_column_id(), idx); + _auto_increment_partial_update_states[idx].init(rowset, schema != nullptr ? schema : tablet->tablet_schema(), + rowset_meta_pb.txn_meta().auto_increment_partial_update_column_id(), + idx); _auto_increment_partial_update_states[idx].src_rss_rowids.resize(_upserts[idx]->size()); auto column = ChunkHelper::column_from_field(*read_column_schema.field(0).get()); @@ -589,12 +589,13 @@ Status RowsetUpdateState::apply(Tablet* tablet, Rowset* rowset, uint32_t rowset_ std::vector read_column_ids; // currently assume it's a partial update (explict for normal, implict for auto increment) if (!txn_meta.partial_update_column_ids().empty()) { - const auto& tschema = tablet->tablet_schema(); + const auto tschema = tablet->tablet_schema(); + // columns supplied in rowset std::vector update_colum_ids(txn_meta.partial_update_column_ids().begin(), txn_meta.partial_update_column_ids().end()); std::set update_columns_set(update_colum_ids.begin(), update_colum_ids.end()); - for (uint32_t i = 0; i < tschema.num_columns(); i++) { + for (uint32_t i = 0; i < tschema->num_columns(); i++) { if (update_columns_set.find(i) == update_columns_set.end()) { read_column_ids.push_back(i); } @@ -612,8 +613,8 @@ Status RowsetUpdateState::apply(Tablet* tablet, Rowset* rowset, uint32_t rowset_ if (txn_meta.has_auto_increment_partial_update_column_id()) { uint32_t id = 0; - for (int i = 0; i < tablet->tablet_schema().num_columns(); ++i) { - if (tablet->tablet_schema().column(i).is_auto_increment()) { + for (int i = 0; i < tablet->tablet_schema()->num_columns(); ++i) { + if (tablet->tablet_schema()->column(i).is_auto_increment()) { id = i; break; } diff --git a/be/src/storage/rowset_update_state.h b/be/src/storage/rowset_update_state.h index ada47617384be..f9fe12d41a34f 100644 --- a/be/src/storage/rowset_update_state.h +++ b/be/src/storage/rowset_update_state.h @@ -56,7 +56,7 @@ struct AutoIncrementPartialUpdateState { std::vector src_rss_rowids; std::unique_ptr write_column; Rowset* rowset; - TabletSchema* schema; + TabletSchemaCSPtr schema; // auto increment column id in partial segment file // but not in full tablet schema uint32_t id; @@ -66,7 +66,7 @@ struct AutoIncrementPartialUpdateState { bool skip_rewrite; AutoIncrementPartialUpdateState() : rowset(nullptr), schema(nullptr), id(0), segment_id(0), skip_rewrite(false) {} - void init(Rowset* rowset, TabletSchema* schema, uint32_t id, uint32_t segment_id) { + void init(Rowset* rowset, TabletSchemaCSPtr schema, uint32_t id, uint32_t segment_id) { this->rowset = rowset; this->schema = schema; this->id = id; diff --git a/be/src/storage/schema_change.cpp b/be/src/storage/schema_change.cpp index 3a61f126057fa..b2f020cf59748 100644 --- a/be/src/storage/schema_change.cpp +++ b/be/src/storage/schema_change.cpp @@ -257,7 +257,8 @@ void ChunkMerger::_pop_heap() { } Status LinkedSchemaChange::process(TabletReader* reader, RowsetWriter* new_rowset_writer, TabletSharedPtr new_tablet, - TabletSharedPtr base_tablet, RowsetSharedPtr rowset) { + TabletSharedPtr base_tablet, RowsetSharedPtr rowset, + TabletSchemaCSPtr base_tablet_schema) { #ifndef BE_TEST Status st = CurrentThread::mem_tracker()->check_mem_limit("LinkedSchemaChange"); if (!st.ok()) { @@ -355,7 +356,7 @@ Status LinkedSchemaChange::generate_delta_column_group_and_cols(const Tablet* ne } } status = chunk_changer->append_generated_columns(read_chunk, new_chunk, all_ref_columns_ids, - base_tablet->tablet_schema().num_columns()); + base_tablet->tablet_schema()->num_columns()); if (!status.ok()) { LOG(WARNING) << "failed to append generated columns"; return Status::InternalError("failed to append generated columns"); @@ -369,7 +370,7 @@ Status LinkedSchemaChange::generate_delta_column_group_and_cols(const Tablet* ne // must record unique column id in delta column group std::vector unique_column_ids; for (const auto& iter : *chunk_changer->get_gc_exprs()) { - ColumnUID unique_id = new_tablet->tablet_schema().column(iter.first).unique_id(); + ColumnUID unique_id = new_tablet->tablet_schema()->column(iter.first).unique_id(); unique_column_ids.emplace_back(unique_id); } std::sort(unique_column_ids.begin(), unique_column_ids.end()); @@ -379,8 +380,7 @@ Status LinkedSchemaChange::generate_delta_column_group_and_cols(const Tablet* ne WritableFileOptions opts{.sync_on_close = true}; ASSIGN_OR_RETURN(auto wfile, fs->new_writable_file(opts, path)); SegmentWriterOptions writer_options; - auto segment_writer = - std::make_unique(std::move(wfile), idx, cols_file_schema.get(), writer_options); + auto segment_writer = std::make_unique(std::move(wfile), idx, cols_file_schema, writer_options); RETURN_IF_ERROR(segment_writer->init(false)); uint64_t segment_file_size = 0; @@ -412,9 +412,11 @@ Status LinkedSchemaChange::generate_delta_column_group_and_cols(const Tablet* ne } Status SchemaChangeDirectly::process(TabletReader* reader, RowsetWriter* new_rowset_writer, TabletSharedPtr new_tablet, - TabletSharedPtr base_tablet, RowsetSharedPtr rowset) { + TabletSharedPtr base_tablet, RowsetSharedPtr rowset, + TabletSchemaCSPtr base_tablet_schema) { + auto cur_base_tablet_schema = !base_tablet_schema ? base_tablet->tablet_schema() : base_tablet_schema; Schema base_schema = - ChunkHelper::convert_schema(base_tablet->tablet_schema(), _chunk_changer->get_selected_column_indexes()); + ChunkHelper::convert_schema(cur_base_tablet_schema, _chunk_changer->get_selected_column_indexes()); ChunkPtr base_chunk = ChunkHelper::new_chunk(base_schema, config::vector_chunk_size); Schema new_schema = ChunkHelper::convert_schema(new_tablet->tablet_schema()); auto char_field_indexes = ChunkHelper::get_char_field_indexes(new_schema); @@ -502,11 +504,12 @@ SchemaChangeWithSorting::SchemaChangeWithSorting(ChunkChanger* chunk_changer, si : SchemaChange(), _chunk_changer(chunk_changer), _memory_limitation(memory_limitation) {} Status SchemaChangeWithSorting::process(TabletReader* reader, RowsetWriter* new_rowset_writer, - TabletSharedPtr new_tablet, TabletSharedPtr base_tablet, - RowsetSharedPtr rowset) { + TabletSharedPtr new_tablet, TabletSharedPtr base_tablet, RowsetSharedPtr rowset, + TabletSchemaCSPtr base_tablet_schema) { + auto cur_base_tablet_schema = !base_tablet_schema ? base_tablet->tablet_schema() : base_tablet_schema; MemTableRowsetWriterSink mem_table_sink(new_rowset_writer); Schema base_schema = - ChunkHelper::convert_schema(base_tablet->tablet_schema(), _chunk_changer->get_selected_column_indexes()); + ChunkHelper::convert_schema(cur_base_tablet_schema, _chunk_changer->get_selected_column_indexes()); Schema new_schema = ChunkHelper::convert_schema(new_tablet->tablet_schema()); auto char_field_indexes = ChunkHelper::get_char_field_indexes(new_schema); @@ -604,9 +607,9 @@ Status SchemaChangeWithSorting::_internal_sorting(std::vector& chunk_a return st; } - std::vector sort_key_idxes = tablet->tablet_schema().sort_key_idxes(); + std::vector sort_key_idxes = tablet->tablet_schema()->sort_key_idxes(); if (sort_key_idxes.empty()) { - sort_key_idxes.resize(tablet->tablet_schema().num_key_columns()); + sort_key_idxes.resize(tablet->tablet_schema()->num_key_columns()); std::iota(sort_key_idxes.begin(), sort_key_idxes.end(), 0); } ChunkMerger merger(std::move(tablet), std::move(sort_key_idxes)); @@ -690,10 +693,22 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& strings::Substitute("tablet $0 is doing disk balance", new_tablet->tablet_id())); } + // Create a new tablet schema, should merge with dropped columns in light weight schema change + TabletSchemaSPtr base_tablet_schema = std::make_shared(); + base_tablet_schema->copy_from(base_tablet->tablet_schema()); + if (!request.columns.empty() && request.columns[0].col_unique_id >= 0) { + base_tablet_schema->clear_columns(); + for (const auto& column : request.columns) { + base_tablet_schema->append_column(TabletColumn(column)); + } + } + SchemaChangeParams sc_params; sc_params.base_tablet = base_tablet; sc_params.new_tablet = new_tablet; - sc_params.chunk_changer = std::make_unique(sc_params.new_tablet->tablet_schema()); + auto tablet_schema_ptr = new_tablet->tablet_schema(); + sc_params.chunk_changer = std::make_unique(tablet_schema_ptr); + sc_params.base_tablet_schema = base_tablet_schema; if (request.__isset.materialized_view_params && request.materialized_view_params.size() > 0) { if (!request.__isset.query_options || !request.__isset.query_globals) { @@ -714,7 +729,7 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& // primary key do not support materialized view, initialize materialized_params_map here, // just for later column_mapping of _parse_request. SchemaChangeUtils::init_materialized_params(request, &sc_params.materialized_params_map); - Status status = SchemaChangeUtils::parse_request(base_tablet->tablet_schema(), new_tablet->tablet_schema(), + Status status = SchemaChangeUtils::parse_request(base_tablet_schema, tablet_schema_ptr, sc_params.chunk_changer.get(), sc_params.materialized_params_map, !base_tablet->delete_predicates().empty(), &sc_params.sc_sorting, &sc_params.sc_directly, &generated_column_idxs); @@ -745,15 +760,15 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& } if (base_tablet->keys_type() == KeysType::PRIMARY_KEYS) { - const auto& base_sort_key_idxes = base_tablet->tablet_schema().sort_key_idxes(); - const auto& new_sort_key_idxes = new_tablet->tablet_schema().sort_key_idxes(); + const auto& base_sort_key_idxes = base_tablet->tablet_schema()->sort_key_idxes(); + const auto& new_sort_key_idxes = new_tablet->tablet_schema()->sort_key_idxes(); std::vector base_sort_key_unique_ids; std::vector new_sort_key_unique_ids; for (auto idx : base_sort_key_idxes) { - base_sort_key_unique_ids.emplace_back(base_tablet->tablet_schema().column(idx).unique_id()); + base_sort_key_unique_ids.emplace_back(base_tablet->tablet_schema()->column(idx).unique_id()); } for (auto idx : new_sort_key_idxes) { - new_sort_key_unique_ids.emplace_back(new_tablet->tablet_schema().column(idx).unique_id()); + new_sort_key_unique_ids.emplace_back(new_tablet->tablet_schema()->column(idx).unique_id()); } if (std::mismatch(new_sort_key_unique_ids.begin(), new_sort_key_unique_ids.end(), base_sort_key_unique_ids.begin()) @@ -828,7 +843,7 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2_normal(const TAlterTable } // prepare tablet reader to prevent rowsets being compacted std::unique_ptr tablet_reader = - std::make_unique(base_tablet, version, base_schema); + std::make_unique(base_tablet, version, base_schema, sc_params.base_tablet_schema); RETURN_IF_ERROR(tablet_reader->prepare()); readers.emplace_back(std::move(tablet_reader)); } @@ -989,7 +1004,7 @@ Status SchemaChangeHandler::_convert_historical_rowsets(SchemaChangeParams& sc_p writer_context.partition_id = new_tablet->partition_id(); writer_context.tablet_schema_hash = new_tablet->schema_hash(); writer_context.rowset_path_prefix = new_tablet->schema_hash_path(); - writer_context.tablet_schema = &new_tablet->tablet_schema(); + writer_context.tablet_schema = new_tablet->tablet_schema(); writer_context.rowset_state = VISIBLE; writer_context.version = sc_params.rowsets_to_change[i]->version(); writer_context.segments_overlap = sc_params.rowsets_to_change[i]->rowset_meta()->segments_overlap(); @@ -1005,7 +1020,7 @@ Status SchemaChangeHandler::_convert_historical_rowsets(SchemaChangeParams& sc_p } auto st = sc_procedure->process(sc_params.rowset_readers[i].get(), rowset_writer.get(), new_tablet, base_tablet, - sc_params.rowsets_to_change[i]); + sc_params.rowsets_to_change[i], sc_params.base_tablet_schema); if (!st.ok()) { LOG(WARNING) << _alter_msg_header << "failed to process the schema change. from tablet " << base_tablet->get_tablet_info().to_string() << " to tablet " diff --git a/be/src/storage/schema_change.h b/be/src/storage/schema_change.h index e1d98e6f6e357..9878c3885a2c1 100644 --- a/be/src/storage/schema_change.h +++ b/be/src/storage/schema_change.h @@ -73,7 +73,8 @@ class SchemaChange { virtual ~SchemaChange() = default; virtual Status process(TabletReader* reader, RowsetWriter* new_rowset_writer, TabletSharedPtr tablet, - TabletSharedPtr base_tablet, RowsetSharedPtr rowset) = 0; + TabletSharedPtr base_tablet, RowsetSharedPtr rowset, + TabletSchemaCSPtr base_tablet_schema = nullptr) = 0; void set_alter_msg_header(std::string msg) { _alter_msg_header = msg; } std::string alter_msg_header() { return _alter_msg_header; } @@ -86,7 +87,8 @@ class LinkedSchemaChange : public SchemaChange { ~LinkedSchemaChange() override = default; Status process(TabletReader* reader, RowsetWriter* new_rowset_writer, TabletSharedPtr new_tablet, - TabletSharedPtr base_tablet, RowsetSharedPtr rowset) override; + TabletSharedPtr base_tablet, RowsetSharedPtr rowset, + TabletSchemaCSPtr base_tablet_schema = nullptr) override; static Status generate_delta_column_group_and_cols(const Tablet* new_tablet, const Tablet* base_tablet, const RowsetSharedPtr& src_rowset, RowsetId rid, int64_t version, @@ -105,7 +107,8 @@ class SchemaChangeDirectly final : public SchemaChange { ~SchemaChangeDirectly() override = default; Status process(TabletReader* reader, RowsetWriter* new_rowset_writer, TabletSharedPtr new_tablet, - TabletSharedPtr base_tablet, RowsetSharedPtr rowset) override; + TabletSharedPtr base_tablet, RowsetSharedPtr rowset, + TabletSchemaCSPtr base_tablet_schema = nullptr) override; private: ChunkChanger* _chunk_changer = nullptr; @@ -119,7 +122,8 @@ class SchemaChangeWithSorting : public SchemaChange { ~SchemaChangeWithSorting() override = default; Status process(TabletReader* reader, RowsetWriter* new_rowset_writer, TabletSharedPtr new_tablet, - TabletSharedPtr base_tablet, RowsetSharedPtr rowset) override; + TabletSharedPtr base_tablet, RowsetSharedPtr rowset, + TabletSchemaCSPtr base_tablet_schema = nullptr) override; static Status _internal_sorting(std::vector& chunk_arr, RowsetWriter* new_rowset_writer, TabletSharedPtr tablet); @@ -146,6 +150,7 @@ class SchemaChangeHandler { TabletSharedPtr new_tablet; std::vector> rowset_readers; Version version; + TabletSchemaCSPtr base_tablet_schema = nullptr; MaterializedViewParamMap materialized_params_map; std::vector rowsets_to_change; bool sc_sorting = false; diff --git a/be/src/storage/schema_change_utils.cpp b/be/src/storage/schema_change_utils.cpp index 4b0e22ab53d6e..a5904dc766be8 100644 --- a/be/src/storage/schema_change_utils.cpp +++ b/be/src/storage/schema_change_utils.cpp @@ -25,8 +25,8 @@ namespace starrocks { -ChunkChanger::ChunkChanger(const TabletSchema& tablet_schema) { - _schema_mapping.resize(tablet_schema.num_columns()); +ChunkChanger::ChunkChanger(const TabletSchemaCSPtr& tablet_schema) { + _schema_mapping.resize(tablet_schema->num_columns()); } ChunkChanger::~ChunkChanger() { @@ -249,13 +249,13 @@ bool ChunkChanger::change_chunk(ChunkPtr& base_chunk, ChunkPtr& new_chunk, const } new_chunk->columns()[i] = std::move(new_col); } else { - LogicalType ref_type = base_tablet_meta->tablet_schema().column(ref_column).type(); - LogicalType new_type = new_tablet_meta->tablet_schema().column(i).type(); + LogicalType ref_type = base_tablet_meta->tablet_schema_ptr()->column(ref_column).type(); + LogicalType new_type = new_tablet_meta->tablet_schema_ptr()->column(i).type(); - int reftype_precision = base_tablet_meta->tablet_schema().column(ref_column).precision(); - int reftype_scale = base_tablet_meta->tablet_schema().column(ref_column).scale(); - int newtype_precision = new_tablet_meta->tablet_schema().column(i).precision(); - int newtype_scale = new_tablet_meta->tablet_schema().column(i).scale(); + int reftype_precision = base_tablet_meta->tablet_schema_ptr()->column(ref_column).precision(); + int reftype_scale = base_tablet_meta->tablet_schema_ptr()->column(ref_column).scale(); + int newtype_precision = new_tablet_meta->tablet_schema_ptr()->column(i).precision(); + int newtype_scale = new_tablet_meta->tablet_schema_ptr()->column(i).scale(); ColumnPtr& base_col = base_chunk->get_column_by_index(ref_column); ColumnPtr& new_col = new_chunk->get_column_by_index(i); @@ -274,7 +274,7 @@ bool ChunkChanger::change_chunk(ChunkPtr& base_chunk, ChunkPtr& new_chunk, const } Slice base_slice = base_datum.get_slice(); Slice slice; - slice.size = new_tablet_meta->tablet_schema().column(i).length(); + slice.size = new_tablet_meta->tablet_schema_ptr()->column(i).length(); slice.data = reinterpret_cast(mem_pool->allocate(slice.size)); if (slice.data == nullptr) { LOG(WARNING) << "failed to allocate memory in mem_pool"; @@ -300,9 +300,9 @@ bool ChunkChanger::change_chunk(ChunkPtr& base_chunk, ChunkPtr& new_chunk, const return false; } - Field ref_field = ChunkHelper::convert_field(ref_column, - base_tablet_meta->tablet_schema().column(ref_column)); - Field new_field = ChunkHelper::convert_field(i, new_tablet_meta->tablet_schema().column(i)); + Field ref_field = ChunkHelper::convert_field( + ref_column, base_tablet_meta->tablet_schema_ptr()->column(ref_column)); + Field new_field = ChunkHelper::convert_field(i, new_tablet_meta->tablet_schema_ptr()->column(i)); Status st = converter->convert_column(ref_field.type().get(), *base_col, new_field.type().get(), new_col.get(), mem_pool); @@ -342,7 +342,7 @@ bool ChunkChanger::change_chunk(ChunkPtr& base_chunk, ChunkPtr& new_chunk, const } if (new_type < ref_type) { LOG(INFO) << "type degraded while altering column. " - << "column=" << new_tablet_meta->tablet_schema().column(i).name() + << "column=" << new_tablet_meta->tablet_schema_ptr()->column(i).name() << ", origin_type=" << logical_type_to_string(ref_type) << ", alter_type=" << logical_type_to_string(new_type) << ", base_tablet_id=" << base_tablet_meta->tablet_id() @@ -617,15 +617,15 @@ void SchemaChangeUtils::init_materialized_params(const TAlterTabletReqV2& reques } } -Status SchemaChangeUtils::parse_request(const TabletSchema& base_schema, const TabletSchema& new_schema, +Status SchemaChangeUtils::parse_request(const TabletSchemaCSPtr& base_schema, const TabletSchemaCSPtr& new_schema, ChunkChanger* chunk_changer, const MaterializedViewParamMap& materialized_view_param_map, bool has_delete_predicates, bool* sc_sorting, bool* sc_directly, std::unordered_set* generated_column_idxs) { std::map base_to_new; bool is_modify_generated_column = false; - for (int i = 0; i < new_schema.num_columns(); ++i) { - const TabletColumn& new_column = new_schema.column(i); + for (int i = 0; i < new_schema->num_columns(); ++i) { + const TabletColumn& new_column = new_schema->column(i); std::string column_name(new_column.name()); ColumnMapping* column_mapping = chunk_changer->get_mutable_column_mapping(i); @@ -643,7 +643,7 @@ Status SchemaChangeUtils::parse_request(const TabletSchema& base_schema, const T RETURN_IF_ERROR(column_mapping->mv_expr_ctx->open(runtime_state)); } - int32_t column_index = base_schema.field_index(mvParam.origin_column_name); + int32_t column_index = base_schema->field_index(mvParam.origin_column_name); if (column_index >= 0) { column_mapping->ref_column = column_index; base_to_new[column_index] = i; @@ -656,7 +656,7 @@ Status SchemaChangeUtils::parse_request(const TabletSchema& base_schema, const T } } - int32_t column_index = base_schema.field_index(column_name); + int32_t column_index = base_schema->field_index(column_name); // if generated_column_idxs contain column_index, it means that // MODIFY GENERATED COLUMN is executed. The value for the new schema // must be re-compute by the new expression so the column mapping can not be set. @@ -676,7 +676,7 @@ Status SchemaChangeUtils::parse_request(const TabletSchema& base_schema, const T { column_mapping->ref_column = -1; - if (i < base_schema.num_short_key_columns()) { + if (i < base_schema->num_short_key_columns()) { *sc_directly = true; } @@ -699,7 +699,7 @@ Status SchemaChangeUtils::parse_request(const TabletSchema& base_schema, const T // If the reference sequence of the Key column is out of order, it needs to be reordered int num_default_value = 0; - for (int i = 0; i < new_schema.num_key_columns(); ++i) { + for (int i = 0; i < new_schema->num_key_columns(); ++i) { ColumnMapping* column_mapping = chunk_changer->get_mutable_column_mapping(i); if (column_mapping->ref_column < 0) { @@ -713,7 +713,7 @@ Status SchemaChangeUtils::parse_request(const TabletSchema& base_schema, const T } } - if (base_schema.keys_type() != new_schema.keys_type()) { + if (base_schema->keys_type() != new_schema->keys_type()) { // only when base table is dup and mv is agg // the rollup job must be reagg. *sc_sorting = true; @@ -727,7 +727,8 @@ Status SchemaChangeUtils::parse_request(const TabletSchema& base_schema, const T // followings need resort: // old keys: A B C D // new keys: A B - if (new_schema.keys_type() != KeysType::DUP_KEYS && new_schema.num_key_columns() < base_schema.num_key_columns()) { + if (new_schema->keys_type() != KeysType::DUP_KEYS && + new_schema->num_key_columns() < base_schema->num_key_columns()) { // this is a table with aggregate key type, and num of key columns in new schema // is less, which means the data in new tablet should be more aggregated. // so we use sorting schema change to sort and merge the data. @@ -735,19 +736,19 @@ Status SchemaChangeUtils::parse_request(const TabletSchema& base_schema, const T return Status::OK(); } - if (base_schema.num_short_key_columns() != new_schema.num_short_key_columns()) { + if (base_schema->num_short_key_columns() != new_schema->num_short_key_columns()) { // the number of short_keys changed, can't do linked schema change *sc_directly = true; return Status::OK(); } - for (size_t i = 0; i < new_schema.num_columns(); ++i) { + for (size_t i = 0; i < new_schema->num_columns(); ++i) { ColumnMapping* column_mapping = chunk_changer->get_mutable_column_mapping(i); if (column_mapping->ref_column < 0) { continue; } else { - auto& new_column = new_schema.column(i); - auto& ref_column = base_schema.column(column_mapping->ref_column); + auto& new_column = new_schema->column(i); + auto& ref_column = base_schema->column(column_mapping->ref_column); if (new_column.type() != ref_column.type() || is_modify_generated_column) { *sc_directly = true; return Status::OK(); diff --git a/be/src/storage/schema_change_utils.h b/be/src/storage/schema_change_utils.h index ef96bd628cfd2..bf2b7b8be627d 100644 --- a/be/src/storage/schema_change_utils.h +++ b/be/src/storage/schema_change_utils.h @@ -32,7 +32,7 @@ using MaterializedViewParamMap = std::unordered_map* materialized_column_idxs); diff --git a/be/src/storage/seek_tuple.h b/be/src/storage/seek_tuple.h index b57b6a72e340b..0af038b734052 100644 --- a/be/src/storage/seek_tuple.h +++ b/be/src/storage/seek_tuple.h @@ -86,7 +86,8 @@ inline std::string SeekTuple::short_key_encode(size_t num_short_keys, uint8_t pa inline std::string SeekTuple::short_key_encode(size_t num_short_keys, std::vector sort_key_idxes, uint8_t padding) const { std::string output; - DCHECK(num_short_keys <= sort_key_idxes.size()); + DCHECK(num_short_keys <= sort_key_idxes.size()) + << "num short key: " << num_short_keys << " vs " << sort_key_idxes.size(); size_t n = std::min(num_short_keys, _values.size()); size_t val_num = _values.size(); for (auto i = 0; i < n; i++) { diff --git a/be/src/storage/snapshot_manager.cpp b/be/src/storage/snapshot_manager.cpp index 2d53e174e72a8..267dafa0c29b7 100644 --- a/be/src/storage/snapshot_manager.cpp +++ b/be/src/storage/snapshot_manager.cpp @@ -198,7 +198,7 @@ Status SnapshotManager::convert_rowset_ids(const string& clone_dir, int64_t tabl // equal to tablet id in meta new_tablet_meta_pb.set_tablet_id(tablet_id); new_tablet_meta_pb.set_schema_hash(schema_hash); - TabletSchema tablet_schema(new_tablet_meta_pb.schema()); + auto tablet_schema = std::make_shared(new_tablet_meta_pb.schema()); std::unordered_map old_to_new_rowsetid; @@ -255,12 +255,12 @@ Status SnapshotManager::convert_rowset_ids(const string& clone_dir, int64_t tabl } Status SnapshotManager::_rename_rowset_id(const RowsetMetaPB& rs_meta_pb, const string& new_path, - TabletSchema& tablet_schema, const RowsetId& rowset_id, + TabletSchemaCSPtr& tablet_schema, const RowsetId& rowset_id, RowsetMetaPB* new_rs_meta_pb) { // TODO use factory to obtain RowsetMeta when SnapshotManager::convert_rowset_ids supports rowset auto rowset_meta = std::make_shared(rs_meta_pb); RowsetSharedPtr org_rowset; - if (!RowsetFactory::create_rowset(&tablet_schema, new_path, rowset_meta, &org_rowset).ok()) { + if (!RowsetFactory::create_rowset(tablet_schema, new_path, rowset_meta, &org_rowset).ok()) { return Status::RuntimeError("fail to create rowset"); } // do not use cache to load index @@ -274,7 +274,7 @@ Status SnapshotManager::_rename_rowset_id(const RowsetMetaPB& rs_meta_pb, const context.partition_id = org_rowset_meta->partition_id(); context.tablet_schema_hash = org_rowset_meta->tablet_schema_hash(); context.rowset_path_prefix = new_path; - context.tablet_schema = &tablet_schema; + context.tablet_schema = org_rowset_meta->tablet_schema() ? org_rowset_meta->tablet_schema() : tablet_schema; context.rowset_state = org_rowset_meta->rowset_state(); context.version = org_rowset_meta->version(); // keep segments_overlap same as origin rowset diff --git a/be/src/storage/snapshot_manager.h b/be/src/storage/snapshot_manager.h index 14292412e846f..1c83c5a0f373d 100644 --- a/be/src/storage/snapshot_manager.h +++ b/be/src/storage/snapshot_manager.h @@ -114,7 +114,7 @@ class SnapshotManager { std::string _get_header_full_path(const TabletSharedPtr& ref_tablet, const std::string& schema_hash_path) const; - Status _rename_rowset_id(const RowsetMetaPB& rs_meta_pb, const string& new_path, TabletSchema& tablet_schema, + Status _rename_rowset_id(const RowsetMetaPB& rs_meta_pb, const string& new_path, TabletSchemaCSPtr& tablet_schema, const RowsetId& next_id, RowsetMetaPB* new_rs_meta_pb); static SnapshotManager* _s_instance; diff --git a/be/src/storage/tablet.cpp b/be/src/storage/tablet.cpp index ca5ed96039a8b..6ea22b43dd811 100644 --- a/be/src/storage/tablet.cpp +++ b/be/src/storage/tablet.cpp @@ -83,6 +83,19 @@ Tablet::Tablet(const TabletMetaSharedPtr& tablet_meta, DataDir* data_dir) _cumulative_point(kInvalidCumulativePoint) { // change _rs_graph to _timestamped_version_tracker _timestamped_version_tracker.construct_versioned_tracker(_tablet_meta->all_rs_metas()); + + // if !_tablet_meta->all_rs_metas()[0]->tablet_schema(), + // that mean the tablet_meta is still no upgrade to support-light-schema-change versions. + // Before support-light-schema-change version, rowset metas don't have tablet schema. + // And when upgrade to starrocks support-light-schema-change version, + // all rowset metas will be set the tablet schema from tablet meta. + if (_tablet_meta->all_rs_metas().empty() || !_tablet_meta->all_rs_metas()[0]->tablet_schema()) { + _max_version_schema = BaseTablet::tablet_schema(); + } else { + _max_version_schema = + TabletMeta::rowset_meta_with_max_rowset_version(_tablet_meta->all_rs_metas())->tablet_schema(); + } + MEM_TRACKER_SAFE_CONSUME(GlobalEnv::GetInstance()->tablet_metadata_mem_tracker(), _mem_usage()); } @@ -114,7 +127,7 @@ Status Tablet::_init_once_action() { for (const auto& rs_meta : _tablet_meta->all_rs_metas()) { Version version = rs_meta->version(); RowsetSharedPtr rowset; - auto st = RowsetFactory::create_rowset(&_tablet_meta->tablet_schema(), _tablet_path, rs_meta, &rowset); + auto st = RowsetFactory::create_rowset(_tablet_meta->tablet_schema_ptr(), _tablet_path, rs_meta, &rowset); if (!st.ok()) { LOG(WARNING) << "fail to init rowset. tablet_id=" << tablet_id() << ", schema_hash=" << schema_hash() << ", version=" << version << ", res=" << st; @@ -128,7 +141,8 @@ Status Tablet::_init_once_action() { Version version = inc_rs_meta->version(); RowsetSharedPtr rowset = get_rowset_by_version(version); if (rowset == nullptr) { - auto st = RowsetFactory::create_rowset(&_tablet_meta->tablet_schema(), _tablet_path, inc_rs_meta, &rowset); + auto st = + RowsetFactory::create_rowset(_tablet_meta->tablet_schema_ptr(), _tablet_path, inc_rs_meta, &rowset); if (!st.ok()) { LOG(WARNING) << "fail to init incremental rowset. tablet_id:" << tablet_id() << ", schema_hash:" << schema_hash() << ", version=" << version << ", res=" << st; @@ -142,7 +156,7 @@ Status Tablet::_init_once_action() { std::shared_ptr row_fetcher = std::make_shared(*this); _binlog_manager = std::make_unique(tablet_id(), schema_hash_path(), config::binlog_file_max_size, config::binlog_page_max_size, - tablet_schema().compression_type(), row_fetcher); + tablet_schema()->compression_type(), row_fetcher); } return Status::OK(); } @@ -216,7 +230,7 @@ Status Tablet::revise_tablet_meta(const std::vector& rowset for (auto& rs_meta : rowsets_to_clone) { Version version = {rs_meta->start_version(), rs_meta->end_version()}; RowsetSharedPtr rowset; - st = RowsetFactory::create_rowset(&_tablet_meta->tablet_schema(), _tablet_path, rs_meta, &rowset); + st = RowsetFactory::create_rowset(_tablet_meta->tablet_schema_ptr(), _tablet_path, rs_meta, &rowset); if (!st.ok()) { LOG(WARNING) << "fail to init rowset. version=" << version; return st; @@ -1652,4 +1666,25 @@ std::string Tablet::debug_string() const { return {}; } +const TabletSchemaCSPtr Tablet::tablet_schema() const { + std::shared_lock rdlock(_schema_lock); + return _max_version_schema; +} + +const TabletSchemaCSPtr Tablet::thread_safe_get_tablet_schema() const { + return _max_version_schema; +} + +void Tablet::update_max_version_schema(const TabletSchemaSPtr& tablet_schema) { + std::lock_guard wrlock(_schema_lock); + // Double Check for concurrent update + if (!_max_version_schema || tablet_schema->schema_version() > _max_version_schema->schema_version()) { + _max_version_schema = tablet_schema; + } +} + +const TabletSchema& Tablet::unsafe_tablet_schema_ref() const { + std::shared_lock rdlock(_schema_lock); + return *_max_version_schema; +} } // namespace starrocks diff --git a/be/src/storage/tablet.h b/be/src/storage/tablet.h index cf6417cdde700..962a897bb2159 100644 --- a/be/src/storage/tablet.h +++ b/be/src/storage/tablet.h @@ -289,6 +289,14 @@ class Tablet : public BaseTablet { void get_basic_info(TabletBasicInfo& info); + const TabletSchemaCSPtr tablet_schema() const override; + + const TabletSchema& unsafe_tablet_schema_ref() const override; + + const TabletSchemaCSPtr thread_safe_get_tablet_schema() const; + + void update_max_version_schema(const TabletSchemaSPtr& tablet_schema); + // verify all rowsets of current(max) version in this tablet [[nodiscard]] Status verify(); @@ -337,6 +345,9 @@ class Tablet : public BaseTablet { TimestampedVersionTracker _timestamped_version_tracker; + // Max schema_version schema from Rowset or FE + TabletSchemaCSPtr _max_version_schema; + OnceFlag _init_once; // meta store lock is used for prevent 2 threads do checkpoint concurrently // it will be used in econ-mode in the future @@ -351,6 +362,7 @@ class Tablet : public BaseTablet { // explain how these two locks work together. mutable std::shared_mutex _meta_lock; + mutable std::shared_mutex _schema_lock; // A new load job will produce a new rowset, which will be inserted into both _rs_version_map // and _inc_rs_version_map. Only the most recent rowsets are kept in _inc_rs_version_map to // reduce the amount of data that needs to be copied during the clone task. @@ -422,27 +434,27 @@ inline void Tablet::set_cumulative_layer_point(int64_t new_point) { } inline KeysType Tablet::keys_type() const { - return _tablet_meta->tablet_schema().keys_type(); + return tablet_schema()->keys_type(); } inline size_t Tablet::num_columns() const { - return _tablet_meta->tablet_schema().num_columns(); + return tablet_schema()->num_columns(); } inline size_t Tablet::num_key_columns() const { - return _tablet_meta->tablet_schema().num_key_columns(); + return tablet_schema()->num_key_columns(); } inline size_t Tablet::num_rows_per_row_block() const { - return _tablet_meta->tablet_schema().num_rows_per_row_block(); + return tablet_schema()->num_rows_per_row_block(); } inline size_t Tablet::next_unique_id() const { - return _tablet_meta->tablet_schema().next_column_unique_id(); + return tablet_schema()->next_column_unique_id(); } inline size_t Tablet::field_index(const string& field_name) const { - return _tablet_meta->tablet_schema().field_index(field_name); + return tablet_schema()->field_index(field_name); } inline bool Tablet::enable_shortcut_compaction() const { diff --git a/be/src/storage/tablet_manager.cpp b/be/src/storage/tablet_manager.cpp index 04401607fded4..ce049d95458fb 100644 --- a/be/src/storage/tablet_manager.cpp +++ b/be/src/storage/tablet_manager.cpp @@ -1267,7 +1267,7 @@ Status TabletManager::_create_inital_rowset_unlocked(const TCreateTabletReq& req context.partition_id = tablet->partition_id(); context.tablet_schema_hash = tablet->schema_hash(); context.rowset_path_prefix = tablet->schema_hash_path(); - context.tablet_schema = &tablet->tablet_schema(); + context.tablet_schema = tablet->tablet_schema(); context.rowset_state = VISIBLE; context.version = version; // there is no data in init rowset, so overlapping info is unknown. @@ -1331,16 +1331,16 @@ Status TabletManager::_create_tablet_meta_unlocked(const TCreateTabletReq& reque // to the new column size_t old_col_idx = 0; for (old_col_idx = 0; old_col_idx < old_num_columns; ++old_col_idx) { - auto old_name = base_tablet->tablet_schema().column(old_col_idx).name(); + auto old_name = base_tablet->tablet_schema()->column(old_col_idx).name(); if (old_name == column.column_name) { - uint32_t old_unique_id = base_tablet->tablet_schema().column(old_col_idx).unique_id(); + uint32_t old_unique_id = base_tablet->tablet_schema()->column(old_col_idx).unique_id(); col_idx_to_unique_id[new_col_idx] = old_unique_id; // During linked schema change, the now() default value is stored in TabletMeta. // When receiving a new schema change request, the last default value stored should be // remained instead of changing. - if (base_tablet->tablet_schema().column(old_col_idx).has_default_value()) { + if (base_tablet->tablet_schema()->column(old_col_idx).has_default_value()) { normal_request.tablet_schema.columns[new_col_idx].__set_default_value( - base_tablet->tablet_schema().column(old_col_idx).default_value()); + base_tablet->tablet_schema()->column(old_col_idx).default_value()); } break; } diff --git a/be/src/storage/tablet_meta.cpp b/be/src/storage/tablet_meta.cpp index 320a283e4b58e..0cdc9ee1d6a0c 100644 --- a/be/src/storage/tablet_meta.cpp +++ b/be/src/storage/tablet_meta.cpp @@ -44,6 +44,7 @@ #include "storage/tablet_meta_manager.h" #include "storage/tablet_schema_map.h" #include "storage/tablet_updates.h" +#include "storage/utils.h" #include "util/uid_util.h" #include "util/url_coding.h" @@ -56,7 +57,7 @@ Status TabletMeta::create(const TCreateTabletReq& request, const TabletUid& tabl *tablet_meta = std::make_shared( request.table_id, request.partition_id, request.tablet_id, request.tablet_schema.schema_hash, shard_id, request.tablet_schema, next_unique_id, - request.__isset.enable_persistent_index ? request.enable_persistent_index : false, col_ordinal_to_unique_id, + request.__isset.enable_persistent_index && request.enable_persistent_index, col_ordinal_to_unique_id, tablet_uid, request.__isset.tablet_type ? request.tablet_type : TTabletType::TABLET_TYPE_DISK, request.__isset.compression_type ? request.compression_type : TCompressionType::LZ4_FRAME); @@ -73,6 +74,14 @@ TabletMetaSharedPtr TabletMeta::create() { return std::make_shared(); } +RowsetMetaSharedPtr& TabletMeta::rowset_meta_with_max_rowset_version(std::vector rowsets) { + return *std::max_element( + rowsets.begin(), rowsets.end(), [](const RowsetMetaSharedPtr& a, const RowsetMetaSharedPtr& b) { + return !a->tablet_schema() || (b->tablet_schema() && a->tablet_schema()->schema_version() < + b->tablet_schema()->schema_version()); + }); +} + TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id, int32_t schema_hash, uint64_t shard_id, const TTabletSchema& tablet_schema, uint32_t next_unique_id, bool enable_persistent_index, diff --git a/be/src/storage/tablet_meta.h b/be/src/storage/tablet_meta.h index 44e0483d5a618..7e592814e58fe 100644 --- a/be/src/storage/tablet_meta.h +++ b/be/src/storage/tablet_meta.h @@ -99,6 +99,8 @@ class TabletMeta { static TabletMetaSharedPtr create(); + static RowsetMetaSharedPtr& rowset_meta_with_max_rowset_version(std::vector rowsets); + explicit TabletMeta(); TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id, int32_t schema_hash, uint64_t shard_id, const TTabletSchema& tablet_schema, uint32_t next_unique_id, bool enable_persistent_index, @@ -153,9 +155,9 @@ class TabletMeta { const TabletSchema& tablet_schema() const; - void set_tablet_schema(const std::shared_ptr& tablet_schema) { _schema = tablet_schema; } + void set_tablet_schema(const TabletSchemaCSPtr& tablet_schema) { _schema = tablet_schema; } - std::shared_ptr& tablet_schema_ptr() { return _schema; } + TabletSchemaCSPtr& tablet_schema_ptr() { return _schema; } const std::vector& all_rs_metas() const; void add_rs_meta(const RowsetMetaSharedPtr& rs_meta); @@ -234,7 +236,7 @@ class TabletMeta { TabletState _tablet_state = TABLET_NOTREADY; // Note: Segment store the pointer of TabletSchema, // so this point should never change - std::shared_ptr _schema = nullptr; + TabletSchemaCSPtr _schema = nullptr; std::vector _rs_metas; std::vector _inc_rs_metas; diff --git a/be/src/storage/tablet_reader.cpp b/be/src/storage/tablet_reader.cpp index c2d145dd48a7c..2f0928a170ff3 100644 --- a/be/src/storage/tablet_reader.cpp +++ b/be/src/storage/tablet_reader.cpp @@ -40,15 +40,20 @@ namespace starrocks { -TabletReader::TabletReader(TabletSharedPtr tablet, const Version& version, Schema schema) - : ChunkIterator(std::move(schema)), _tablet(std::move(tablet)), _version(version) {} +TabletReader::TabletReader(TabletSharedPtr tablet, const Version& version, Schema schema, + const TabletSchemaCSPtr& tablet_schema) + : ChunkIterator(std::move(schema)), _tablet(std::move(tablet)), _version(version) { + _tablet_schema = !tablet_schema ? _tablet->tablet_schema() : tablet_schema; +} TabletReader::TabletReader(TabletSharedPtr tablet, const Version& version, Schema schema, - std::vector captured_rowsets) + std::vector captured_rowsets, const TabletSchemaSPtr* tablet_schema) : ChunkIterator(std::move(schema)), _tablet(std::move(tablet)), _version(version), - _rowsets(std::move(captured_rowsets)) {} + _rowsets(std::move(captured_rowsets)) { + _tablet_schema = tablet_schema ? *tablet_schema : _tablet->tablet_schema(); +} TabletReader::TabletReader(TabletSharedPtr tablet, const Version& version, Schema schema, bool is_key, RowSourceMaskBuffer* mask_buffer) @@ -59,6 +64,13 @@ TabletReader::TabletReader(TabletSharedPtr tablet, const Version& version, Schem _is_key(is_key), _mask_buffer(mask_buffer) { DCHECK(_mask_buffer); + _tablet_schema = _tablet->tablet_schema(); +} + +TabletReader::TabletReader(TabletSharedPtr tablet, const Version& version, const TabletSchemaSPtr& tablet_schema, + Schema schema) + : ChunkIterator(std::move(schema)), _tablet(std::move(tablet)), _version(version) { + _tablet_schema = tablet_schema; } void TabletReader::close() { @@ -70,6 +82,7 @@ void TabletReader::close() { Rowset::release_readers(_rowsets); _rowsets.clear(); _obj_pool.clear(); + _tablet_schema.reset(); } Status TabletReader::prepare() { @@ -89,6 +102,7 @@ Status TabletReader::prepare() { } _stats.rowsets_read_count += _rowsets.size(); Rowset::acquire_readers(_rowsets); + // ensure all input rowsets are loaded into memory for (const auto& rowset : _rowsets) { rowset->load(); @@ -115,7 +129,7 @@ Status TabletReader::_init_collector_for_pk_index_read() { // get pk eq predicates, and convert these predicates to encoded pk column const auto& tablet_schema = _tablet->tablet_schema(); vector pk_column_ids; - for (size_t i = 0; i < tablet_schema.num_key_columns(); i++) { + for (size_t i = 0; i < tablet_schema->num_key_columns(); i++) { pk_column_ids.emplace_back(i); } auto pk_schema = ChunkHelper::convert_schema(tablet_schema, pk_column_ids); @@ -124,7 +138,7 @@ Status TabletReader::_init_collector_for_pk_index_read() { size_t num_pk_eq_predicates = 0; for (const ColumnPredicate* pred : _reader_params->predicates) { auto column_id = pred->column_id(); - if (column_id < tablet_schema.num_key_columns() && pred->type() == PredicateType::kEQ) { + if (column_id < tablet_schema->num_key_columns() && pred->type() == PredicateType::kEQ) { auto& column = keys->get_column_by_id(column_id); if (column->size() != 0) { return Status::NotSupported( @@ -136,15 +150,15 @@ Status TabletReader::_init_collector_for_pk_index_read() { pushdown_predicates[pred->column_id()].emplace_back(pred); } } - if (num_pk_eq_predicates != tablet_schema.num_key_columns()) { + if (num_pk_eq_predicates != tablet_schema->num_key_columns()) { return Status::NotSupported(strings::Substitute("should have eq predicates on all pk columns current: $0 < $1", - num_pk_eq_predicates, tablet_schema.num_key_columns())); + num_pk_eq_predicates, tablet_schema->num_key_columns())); } std::unique_ptr pk_column; - if (!PrimaryKeyEncoder::create_column(*tablet_schema.schema(), &pk_column).ok()) { + if (!PrimaryKeyEncoder::create_column(*tablet_schema->schema(), &pk_column).ok()) { CHECK(false) << "create column for primary key encoder failed tablet_id:" << _tablet->tablet_id(); } - PrimaryKeyEncoder::encode(*tablet_schema.schema(), *keys, 0, keys->num_rows(), pk_column.get()); + PrimaryKeyEncoder::encode(*tablet_schema->schema(), *keys, 0, keys->num_rows(), pk_column.get()); // get rowid using pk index std::vector rowids(1); @@ -180,7 +194,7 @@ Status TabletReader::_init_collector_for_pk_index_read() { rs_opts.runtime_state = _reader_params->runtime_state; rs_opts.profile = _reader_params->profile; rs_opts.use_page_cache = _reader_params->use_page_cache; - rs_opts.tablet_schema = &_tablet->tablet_schema(); + rs_opts.tablet_schema = _tablet->tablet_schema(); rs_opts.global_dictmaps = _reader_params->global_dictmaps; rs_opts.unused_output_column_ids = _reader_params->unused_output_column_ids; rs_opts.runtime_range_pruner = _reader_params->runtime_range_pruner; @@ -236,7 +250,7 @@ Status TabletReader::do_get_next(Chunk* chunk, std::vector* sourc Status TabletReader::get_segment_iterators(const TabletReaderParams& params, std::vector* iters) { RowsetReadOptions rs_opts; - KeysType keys_type = _tablet->tablet_schema().keys_type(); + KeysType keys_type = _tablet_schema->keys_type(); RETURN_IF_ERROR(_init_predicates(params)); RETURN_IF_ERROR(_init_delete_predicates(params, &_delete_predicates)); RETURN_IF_ERROR(parse_seek_range(_tablet, params.range, params.end_range, params.start_key, params.end_key, @@ -252,7 +266,7 @@ Status TabletReader::get_segment_iterators(const TabletReaderParams& params, std rs_opts.runtime_state = params.runtime_state; rs_opts.profile = params.profile; rs_opts.use_page_cache = params.use_page_cache; - rs_opts.tablet_schema = &_tablet->tablet_schema(); + rs_opts.tablet_schema = _tablet_schema; rs_opts.global_dictmaps = params.global_dictmaps; rs_opts.unused_output_column_ids = params.unused_output_column_ids; rs_opts.runtime_range_pruner = params.runtime_range_pruner; @@ -293,10 +307,10 @@ Status TabletReader::_init_collector(const TabletReaderParams& params) { // If |keys_type| is UNIQUE_KEYS and |params.skip_aggregation| is true, must disable aggregate totally. // If |keys_type| is AGG_KEYS and |params.skip_aggregation| is true, aggregate is an optional operation. - KeysType keys_type = _tablet->tablet_schema().keys_type(); + KeysType keys_type = _tablet_schema->keys_type(); const auto skip_aggr = params.skip_aggregation; - const auto select_all_keys = _schema.num_key_fields() == _tablet->num_key_columns(); - DCHECK_LE(_schema.num_key_fields(), _tablet->num_key_columns()); + const auto select_all_keys = _schema.num_key_fields() == _tablet_schema->num_key_columns(); + DCHECK_LE(_schema.num_key_fields(), _tablet_schema->num_key_columns()); if (seg_iters.empty()) { _collect_iter = new_empty_iterator(_schema, params.chunk_size); @@ -446,7 +460,7 @@ Status TabletReader::_init_predicates(const TabletReaderParams& params) { } Status TabletReader::_init_delete_predicates(const TabletReaderParams& params, DeletePredicates* dels) { - PredicateParser pred_parser(_tablet->tablet_schema()); + PredicateParser pred_parser(_tablet_schema); std::shared_lock header_lock(_tablet->get_header_lock()); // here we can not use DeletePredicatePB from _tablet->delete_predicates() because @@ -466,8 +480,8 @@ Status TabletReader::_init_delete_predicates(const TabletReaderParams& params, D LOG(WARNING) << "invalid delete condition: " << pred_pb.sub_predicates(i) << "]"; return Status::InternalError("invalid delete condition string"); } - if (_tablet->tablet_schema().field_index(cond.column_name) >= _tablet->num_key_columns() && - _tablet->keys_type() != DUP_KEYS) { + if (_tablet_schema->field_index(cond.column_name) >= _tablet_schema->num_key_columns() && + _tablet_schema->keys_type() != DUP_KEYS) { LOG(WARNING) << "ignore delete condition of non-key column: " << pred_pb.sub_predicates(i); continue; } @@ -514,12 +528,12 @@ Status TabletReader::_init_delete_predicates(const TabletReaderParams& params, D } // convert an OlapTuple to SeekTuple. -Status TabletReader::_to_seek_tuple(const TabletSchema& tablet_schema, const OlapTuple& input, SeekTuple* tuple, +Status TabletReader::_to_seek_tuple(const TabletSchemaCSPtr& tablet_schema, const OlapTuple& input, SeekTuple* tuple, MemPool* mempool) { Schema schema; std::vector values; values.reserve(input.size()); - const auto& sort_key_idxes = tablet_schema.sort_key_idxes(); + const auto& sort_key_idxes = tablet_schema->sort_key_idxes(); DCHECK(sort_key_idxes.empty() || sort_key_idxes.size() >= input.size()); if (sort_key_idxes.size() > 0) { @@ -529,7 +543,7 @@ Status TabletReader::_to_seek_tuple(const TabletSchema& tablet_schema, const Ola } for (size_t i = 0; i < input.size(); i++) { int idx = sort_key_idxes.empty() ? i : sort_key_idxes[i]; - auto f = std::make_shared(ChunkHelper::convert_field(idx, tablet_schema.column(idx))); + auto f = std::make_shared(ChunkHelper::convert_field(idx, tablet_schema->column(idx))); schema.append(f); values.emplace_back(Datum()); if (input.is_null(i)) { diff --git a/be/src/storage/tablet_reader.h b/be/src/storage/tablet_reader.h index d6b4490647c7c..7cc4519609fd9 100644 --- a/be/src/storage/tablet_reader.h +++ b/be/src/storage/tablet_reader.h @@ -26,6 +26,7 @@ #include "storage/seek_range.h" #include "storage/tablet.h" #include "storage/tablet_reader_params.h" +#include "storage/tablet_schema.h" namespace starrocks { @@ -33,12 +34,14 @@ class ColumnPredicate; class TabletReader final : public ChunkIterator { public: - TabletReader(TabletSharedPtr tablet, const Version& version, Schema schema); + TabletReader(TabletSharedPtr tablet, const Version& version, Schema schema, + const TabletSchemaCSPtr& tablet_schema = nullptr); // *captured_rowsets* is captured forward before creating TabletReader. TabletReader(TabletSharedPtr tablet, const Version& version, Schema schema, - std::vector captured_rowsets); + std::vector captured_rowsets, const TabletSchemaSPtr* tablet_schema = nullptr); TabletReader(TabletSharedPtr tablet, const Version& version, Schema schema, bool is_key, RowSourceMaskBuffer* mask_buffer); + TabletReader(TabletSharedPtr tablet, const Version& version, const TabletSchemaSPtr& tablet_schema, Schema schema); ~TabletReader() override { close(); } Status prepare(); @@ -74,12 +77,13 @@ class TabletReader final : public ChunkIterator { Status _init_delete_predicates(const TabletReaderParams& read_params, DeletePredicates* dels); Status _init_collector(const TabletReaderParams& read_params); - static Status _to_seek_tuple(const TabletSchema& tablet_schema, const OlapTuple& input, SeekTuple* tuple, + static Status _to_seek_tuple(const TabletSchemaCSPtr& tablet_schema, const OlapTuple& input, SeekTuple* tuple, MemPool* mempool); Status _init_collector_for_pk_index_read(); TabletSharedPtr _tablet; + TabletSchemaCSPtr _tablet_schema; Version _version; MemPool _mempool; diff --git a/be/src/storage/tablet_schema.cpp b/be/src/storage/tablet_schema.cpp index 62ae828a635e8..ad28e8730a71b 100644 --- a/be/src/storage/tablet_schema.cpp +++ b/be/src/storage/tablet_schema.cpp @@ -34,14 +34,18 @@ #include "storage/tablet_schema.h" +#include + #include #include #include "runtime/exec_env.h" #include "runtime/mem_tracker.h" #include "storage/chunk_helper.h" +#include "storage/metadata_util.h" #include "storage/tablet_schema_map.h" #include "storage/type_utils.h" +#include "tablet_meta.h" namespace starrocks { @@ -149,6 +153,14 @@ TabletColumn::TabletColumn(TabletColumn&& rhs) noexcept rhs._extra_fields = nullptr; } +TabletColumn::TabletColumn(const ColumnPB& column) { + init_from_pb(column); +} + +TabletColumn::TabletColumn(const TColumn& column) { + init_from_thrift(column); +} + TabletColumn::~TabletColumn() { delete _extra_fields; } @@ -233,6 +245,16 @@ void TabletColumn::init_from_pb(const ColumnPB& column) { } } +void TabletColumn::init_from_thrift(const TColumn& tcolumn) { + _unique_id = tcolumn.col_unique_id; + ColumnPB column_pb; + auto shared_tcolumn_desc = std::make_shared(tcolumn); + convert_to_new_version(shared_tcolumn_desc.get()); + + t_column_to_pb_column(_unique_id, *shared_tcolumn_desc, &column_pb); + init_from_pb(column_pb); +} + void TabletColumn::to_schema_pb(ColumnPB* column) const { column->mutable_name()->assign(_col_name.data(), _col_name.size()); column->set_unique_id(_unique_id); @@ -259,6 +281,32 @@ void TabletColumn::to_schema_pb(ColumnPB* column) const { } } +void TabletSchema::append_column(TabletColumn column) { + if (column.is_key()) { + _num_key_columns++; + } + _field_id_to_index[column.unique_id()] = _num_columns; + _cols.push_back(std::move(column)); + if (_sort_key_idxes_set.count(_num_columns) > 0) { + _cols[_num_columns].set_is_sort_key(true); + } + _num_columns++; +} + +void TabletSchema::clear_columns() { + _field_id_to_index.clear(); + _num_columns = 0; + _num_key_columns = 0; + _cols.clear(); +} + +void TabletSchema::copy_from(const std::shared_ptr& tablet_schema) { + TabletSchemaPB tablet_schema_pb; + tablet_schema->to_schema_pb(&tablet_schema_pb); + _init_from_pb(tablet_schema_pb); + MEM_TRACKER_SAFE_CONSUME(GlobalEnv::GetInstance()->tablet_schema_mem_tracker(), mem_usage()) +} + void TabletColumn::add_sub_column(const TabletColumn& sub_column) { _get_or_alloc_extra_fields()->sub_columns.push_back(sub_column); } @@ -279,36 +327,42 @@ std::shared_ptr TabletSchema::create(const TabletSchemaPB& schema_ return std::make_shared(schema_pb, schema_map); } -std::shared_ptr TabletSchema::create(const TabletSchema& src_tablet_schema, +std::shared_ptr TabletSchema::create(const TabletSchemaCSPtr& src_tablet_schema, const std::vector& referenced_column_ids) { TabletSchemaPB partial_tablet_schema_pb; - partial_tablet_schema_pb.set_id(src_tablet_schema.id()); - partial_tablet_schema_pb.set_next_column_unique_id(src_tablet_schema.next_column_unique_id()); - partial_tablet_schema_pb.set_num_rows_per_row_block(src_tablet_schema.num_rows_per_row_block()); - partial_tablet_schema_pb.set_num_short_key_columns(src_tablet_schema.num_short_key_columns()); - partial_tablet_schema_pb.set_keys_type(src_tablet_schema.keys_type()); - if (src_tablet_schema.has_bf_fpp()) { - partial_tablet_schema_pb.set_bf_fpp(src_tablet_schema.bf_fpp()); + partial_tablet_schema_pb.set_id(src_tablet_schema->id()); + partial_tablet_schema_pb.set_next_column_unique_id(src_tablet_schema->next_column_unique_id()); + partial_tablet_schema_pb.set_num_rows_per_row_block(src_tablet_schema->num_rows_per_row_block()); + partial_tablet_schema_pb.set_num_short_key_columns(src_tablet_schema->num_short_key_columns()); + partial_tablet_schema_pb.set_keys_type(src_tablet_schema->keys_type()); + if (src_tablet_schema->has_bf_fpp()) { + partial_tablet_schema_pb.set_bf_fpp(src_tablet_schema->bf_fpp()); } for (const auto referenced_column_id : referenced_column_ids) { auto* tablet_column = partial_tablet_schema_pb.add_column(); - src_tablet_schema.column(referenced_column_id).to_schema_pb(tablet_column); + src_tablet_schema->column(referenced_column_id).to_schema_pb(tablet_column); } return std::make_shared(partial_tablet_schema_pb); } -std::shared_ptr TabletSchema::create_with_uid(const TabletSchema& tablet_schema, +std::shared_ptr TabletSchema::create_with_uid(const TabletSchemaCSPtr& tablet_schema, const std::vector& unique_column_ids) { std::unordered_set unique_cid_filter(unique_column_ids.begin(), unique_column_ids.end()); std::vector column_indexes; - for (int cid = 0; cid < tablet_schema.columns().size(); cid++) { - if (unique_cid_filter.count(tablet_schema.column(cid).unique_id()) > 0) { + for (int cid = 0; cid < tablet_schema->columns().size(); cid++) { + if (unique_cid_filter.count(tablet_schema->column(cid).unique_id()) > 0) { column_indexes.push_back(cid); } } return TabletSchema::create(tablet_schema, column_indexes); } +std::unique_ptr TabletSchema::copy(const std::shared_ptr& tablet_schema) { + auto t_ptr = std::make_unique(); + t_ptr->copy_from(tablet_schema); + return t_ptr; +} + void TabletSchema::_init_schema() const { starrocks::Fields fields; for (ColumnId cid = 0; cid < num_columns(); ++cid) { @@ -344,6 +398,7 @@ void TabletSchema::_init_from_pb(const TabletSchemaPB& schema) { _id = schema.has_id() ? schema.id() : invalid_id(); _keys_type = static_cast(schema.keys_type()); _num_key_columns = 0; + _num_columns = 0; _cols.clear(); _compression_type = schema.compression_type(); for (auto& column_pb : schema.column()) { @@ -353,16 +408,20 @@ void TabletSchema::_init_from_pb(const TabletSchemaPB& schema) { if (column.is_key()) { _num_key_columns++; } + _field_id_to_index[column.unique_id()] = _num_columns; + _num_columns++; } if (schema.sort_key_idxes().empty()) { _sort_key_idxes.reserve(_num_key_columns); for (auto i = 0; i < _num_key_columns; ++i) { _sort_key_idxes.push_back(i); + _sort_key_idxes_set.emplace(i); } } else { _sort_key_idxes.reserve(schema.sort_key_idxes_size()); for (auto i = 0; i < schema.sort_key_idxes_size(); ++i) { _sort_key_idxes.push_back(schema.sort_key_idxes(i)); + _sort_key_idxes_set.emplace(schema.sort_key_idxes(i)); } } for (auto cid : _sort_key_idxes) { @@ -378,6 +437,53 @@ void TabletSchema::_init_from_pb(const TabletSchemaPB& schema) { _has_bf_fpp = false; _bf_fpp = BLOOM_FILTER_DEFAULT_FPP; } + _schema_version = schema.schema_version(); +} + +void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version, const POlapTableIndexSchema& index, + const TabletSchemaCSPtr& ori_tablet_schema) { + // copy from ori_tablet_schema + _keys_type = ori_tablet_schema->keys_type(); + _num_short_key_columns = ori_tablet_schema->num_short_key_columns(); + _num_rows_per_row_block = ori_tablet_schema->num_rows_per_row_block(); + _compression_type = ori_tablet_schema->compression_type(); + + // todo(yixiu): unique_id + _next_column_unique_id = ori_tablet_schema->next_column_unique_id(); + // copy from table_schema_param + _num_key_columns = 0; + _num_columns = 0; + bool has_bf_columns = false; + _cols.clear(); + _field_id_to_index.clear(); + + _schema_version = version; + if (index.id() == index_id) { + for (auto& pcolumn : index.columns_desc()) { + TabletColumn column; + column.init_from_pb(pcolumn); + if (column.is_key()) { + _num_key_columns++; + } + if (column.is_bf_column()) { + has_bf_columns = true; + } + _field_id_to_index[column.unique_id()] = _num_columns; + _cols.emplace_back(std::move(column)); + _num_columns++; + } + } + + for (auto cid : _sort_key_idxes) { + _cols[cid].set_is_sort_key(true); + } + if (has_bf_columns) { + _has_bf_fpp = true; + _bf_fpp = ori_tablet_schema->bf_fpp(); + } else { + _has_bf_fpp = false; + _bf_fpp = BLOOM_FILTER_DEFAULT_FPP; + } } void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const { @@ -396,6 +502,7 @@ void TabletSchema::to_schema_pb(TabletSchemaPB* tablet_schema_pb) const { tablet_schema_pb->set_next_column_unique_id(_next_column_unique_id); tablet_schema_pb->set_compression_type(_compression_type); tablet_schema_pb->mutable_sort_key_idxes()->Add(_sort_key_idxes.begin(), _sort_key_idxes.end()); + tablet_schema_pb->set_schema_version(_schema_version); } size_t TabletSchema::estimate_row_size(size_t variable_len) const { @@ -417,6 +524,11 @@ size_t TabletSchema::field_index(std::string_view field_name) const { return -1; } +int32_t TabletSchema::field_index(int32_t col_unique_id) const { + const auto& found = _field_id_to_index.find(col_unique_id); + return (found == _field_id_to_index.end()) ? -1 : found->second; +} + const std::vector& TabletSchema::columns() const { return _cols; } diff --git a/be/src/storage/tablet_schema.h b/be/src/storage/tablet_schema.h index fadc958511102..7d77dbd6cfcdf 100644 --- a/be/src/storage/tablet_schema.h +++ b/be/src/storage/tablet_schema.h @@ -53,6 +53,8 @@ namespace starrocks { class TabletSchemaMap; class MemTracker; class SegmentReaderWriterTest; +class POlapTableIndexSchema; +class TColumn; class TabletColumn { struct ExtraFields { @@ -72,6 +74,8 @@ class TabletColumn { using ColumnScale = uint8_t; TabletColumn(); + TabletColumn(const ColumnPB& column); + TabletColumn(const TColumn& column); TabletColumn(StorageAggregateType agg, LogicalType type); TabletColumn(StorageAggregateType agg, LogicalType type, bool is_nullable); TabletColumn(StorageAggregateType agg, LogicalType type, bool is_nullable, int32_t unique_id, size_t length); @@ -87,6 +91,7 @@ class TabletColumn { void swap(TabletColumn* rhs); void init_from_pb(const ColumnPB& column); + void init_from_thrift(const TColumn& column); void to_schema_pb(ColumnPB* column) const; ColumnUID unique_id() const { return _unique_id; } @@ -227,19 +232,21 @@ bool operator!=(const TabletColumn& a, const TabletColumn& b); class TabletSchema { public: using SchemaId = int64_t; + using TabletSchemaCSPtr = std::shared_ptr; static std::shared_ptr create(const TabletSchemaPB& schema_pb); static std::shared_ptr create(const TabletSchemaPB& schema_pb, TabletSchemaMap* schema_map); - static std::shared_ptr create(const TabletSchema& tablet_schema, + static std::shared_ptr create(const TabletSchemaCSPtr& tablet_schema, const std::vector& column_indexes); - static std::shared_ptr create_with_uid(const TabletSchema& tablet_schema, + static std::shared_ptr create_with_uid(const TabletSchemaCSPtr& tablet_schema, const std::vector& unique_column_ids); + static std::unique_ptr copy(const std::shared_ptr& tablet_schema); // Must be consistent with MaterializedIndexMeta.INVALID_SCHEMA_ID defined in // file ./fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedIndexMeta.java constexpr static SchemaId invalid_id() { return 0; } - TabletSchema() = delete; + TabletSchema() = default; explicit TabletSchema(const TabletSchemaPB& schema_pb); // Does NOT take ownership of |schema_map| and |schema_map| must outlive TabletSchema. TabletSchema(const TabletSchemaPB& schema_pb, TabletSchemaMap* schema_map); @@ -251,6 +258,7 @@ class TabletSchema { // Caller should always check the returned value with `invalid_id()`. SchemaId id() const { return _id; } size_t estimate_row_size(size_t variable_len) const; + int32_t field_index(int32_t col_unique_id) const; size_t field_index(std::string_view field_name) const; const TabletColumn& column(size_t ordinal) const; const std::vector& columns() const; @@ -264,6 +272,11 @@ class TabletSchema { bool has_bf_fpp() const { return _has_bf_fpp; } double bf_fpp() const { return _bf_fpp; } CompressionTypePB compression_type() const { return _compression_type; } + void append_column(TabletColumn column); + + int32_t schema_version() const { return _schema_version; } + void clear_columns(); + void copy_from(const std::shared_ptr& tablet_schema); std::string debug_string() const; @@ -279,6 +292,9 @@ class TabletSchema { Schema* schema() const; + void build_current_tablet_schema(int64_t index_id, int32_t version, const POlapTableIndexSchema& index, + const std::shared_ptr& ori_tablet_schema); + private: friend class SegmentReaderWriterTest; FRIEND_TEST(SegmentReaderWriterTest, estimate_segment_size); @@ -300,20 +316,28 @@ class TabletSchema { size_t _num_rows_per_row_block = 0; size_t _next_column_unique_id = 0; - uint16_t _num_key_columns = 0; + mutable uint32_t _num_columns = 0; + mutable uint16_t _num_key_columns = 0; uint16_t _num_short_key_columns = 0; std::vector _sort_key_idxes; + std::unordered_set _sort_key_idxes_set; uint8_t _keys_type = static_cast(DUP_KEYS); CompressionTypePB _compression_type = CompressionTypePB::LZ4_FRAME; + std::unordered_map _field_id_to_index; + bool _has_bf_fpp = false; mutable std::unique_ptr _schema; mutable std::once_flag _init_schema_once_flag; + int32_t _schema_version = -1; }; bool operator==(const TabletSchema& a, const TabletSchema& b); bool operator!=(const TabletSchema& a, const TabletSchema& b); +using TabletSchemaSPtr = std::shared_ptr; +using TabletSchemaCSPtr = std::shared_ptr; + } // namespace starrocks diff --git a/be/src/storage/tablet_schema_map.cpp b/be/src/storage/tablet_schema_map.cpp index e5ac3143a7ba3..ee84287c14ce2 100644 --- a/be/src/storage/tablet_schema_map.cpp +++ b/be/src/storage/tablet_schema_map.cpp @@ -26,8 +26,7 @@ static void get_stats(std::ostream& os, void*) { // NOLINTNEXTLINE bvar::PassiveStatus g_schema_map_stats("tablet_schema_map", get_stats, NULL); -bool TabletSchemaMap::check_schema_unique_id(const TabletSchemaPB& schema_pb, - const std::shared_ptr& schema_ptr) { +bool TabletSchemaMap::check_schema_unique_id(const TabletSchemaPB& schema_pb, const TabletSchemaCSPtr& schema_ptr) { if (schema_pb.next_column_unique_id() != schema_ptr->next_column_unique_id() || schema_pb.column_size() != schema_ptr->num_columns()) { return false; @@ -54,7 +53,7 @@ std::pair TabletSchemaMap::emplace(const // We use shared schema to save mem usage, but the premise is that we need to ensure that no two different // TabletSchemaPBs have the same id. But we can't guarantee it after schema change so far, so we should // check the consistent of tablet schema. - // If check failed, we will create a new tablet_schema as return value, but we must hold the original schema + // If check failed, we will create a new unsafe_tablet_schema_ref as return value, but we must hold the original schema // in map until the shard lock is release. If not, we may be deconstruct the original schema which will cause // a dead lock(#issue 5646) TabletSchemaPtr result = nullptr; diff --git a/be/src/storage/tablet_schema_map.h b/be/src/storage/tablet_schema_map.h index d86a356042b88..1f5dfb7066c52 100644 --- a/be/src/storage/tablet_schema_map.h +++ b/be/src/storage/tablet_schema_map.h @@ -73,7 +73,7 @@ class TabletSchemaMap { private: constexpr static int kShardSize = 16; - bool check_schema_unique_id(const TabletSchemaPB& schema_pb, const std::shared_ptr& schema_ptr); + bool check_schema_unique_id(const TabletSchemaPB& schema_pb, const TabletSchemaCSPtr& schema_ptr); struct MapShard { mutable std::mutex mtx; phmap::flat_hash_map> map; diff --git a/be/src/storage/tablet_updates.cpp b/be/src/storage/tablet_updates.cpp index 31356b43061e4..38e7e621a90b6 100644 --- a/be/src/storage/tablet_updates.cpp +++ b/be/src/storage/tablet_updates.cpp @@ -165,8 +165,9 @@ Status TabletUpdates::_load_rowsets_and_check_consistency(std::set& un RETURN_IF_ERROR(TabletMetaManager::rowset_iterate( _tablet.data_dir(), _tablet.tablet_id(), [&](const RowsetMetaSharedPtr& rowset_meta) -> bool { RowsetSharedPtr rowset; - auto st = RowsetFactory::create_rowset(&_tablet.tablet_schema(), _tablet.schema_hash_path(), - rowset_meta, &rowset); + + auto st = RowsetFactory::create_rowset(_tablet.thread_safe_get_tablet_schema(), + _tablet.schema_hash_path(), rowset_meta, &rowset); if (st.ok()) { _rowsets[rowset_meta->get_rowset_seg_id()] = std::move(rowset); } else { @@ -236,8 +237,8 @@ Status TabletUpdates::_load_pending_rowsets() { auto rowset_meta = std::make_shared(rowset_meta_data, &parse_ok); CHECK(parse_ok) << "Corrupted rowset meta"; RowsetSharedPtr rowset; - auto st = RowsetFactory::create_rowset(&_tablet.tablet_schema(), _tablet.schema_hash_path(), - rowset_meta, &rowset); + auto st = RowsetFactory::create_rowset(_tablet.thread_safe_get_tablet_schema(), + _tablet.schema_hash_path(), rowset_meta, &rowset); if (st.ok()) { _pending_commits.emplace(version, rowset); } else { @@ -1139,8 +1140,8 @@ void TabletUpdates::_apply_normal_rowset_commit(const EditVersionInfo& version_i int32_t conditional_column = -1; const auto& txn_meta = rowset->rowset_meta()->get_meta_pb().txn_meta(); if (txn_meta.has_merge_condition()) { - for (int i = 0; i < _tablet.tablet_schema().columns().size(); ++i) { - if (_tablet.tablet_schema().column(i).name() == txn_meta.merge_condition()) { + for (int i = 0; i < _tablet.thread_safe_get_tablet_schema()->columns().size(); ++i) { + if (_tablet.thread_safe_get_tablet_schema()->column(i).name() == txn_meta.merge_condition()) { conditional_column = i; break; } @@ -1525,7 +1526,7 @@ Status TabletUpdates::_do_update(uint32_t rowset_id, int32_t upsert_idx, int32_t const std::vector& upserts, PrimaryIndex& index, int64_t tablet_id, DeletesMap* new_deletes) { if (condition_column >= 0) { - auto tablet_column = _tablet.tablet_schema().column(condition_column); + auto tablet_column = _tablet.thread_safe_get_tablet_schema()->column(condition_column); std::vector read_column_ids; read_column_ids.push_back(condition_column); @@ -1634,7 +1635,7 @@ Status TabletUpdates::_do_compaction(std::unique_ptr* pinfo) { context.partition_id = _tablet.partition_id(); context.tablet_schema_hash = _tablet.schema_hash(); context.rowset_path_prefix = _tablet.schema_hash_path(); - context.tablet_schema = &_tablet.tablet_schema(); + context.tablet_schema = _tablet.thread_safe_get_tablet_schema(); context.rowset_state = COMMITTED; context.segments_overlap = NONOVERLAPPING; context.max_rows_per_segment = @@ -3228,7 +3229,7 @@ Status TabletUpdates::convert_from(const std::shared_ptr& base_tablet, i uint32_t next_rowset_id = 0; std::vector new_rowset_load_infos(src_rowsets.size()); - Schema base_schema = ChunkHelper::convert_schema(base_tablet->tablet_schema()); + Schema base_schema = ChunkHelper::convert_schema(base_tablet->thread_safe_get_tablet_schema()); OlapReaderStatistics stats; @@ -3254,7 +3255,7 @@ Status TabletUpdates::convert_from(const std::shared_ptr& base_tablet, i writer_context.partition_id = _tablet.partition_id(); writer_context.tablet_schema_hash = _tablet.schema_hash(); writer_context.rowset_path_prefix = _tablet.schema_hash_path(); - writer_context.tablet_schema = &_tablet.tablet_schema(); + writer_context.tablet_schema = _tablet.thread_safe_get_tablet_schema(); writer_context.rowset_state = VISIBLE; writer_context.version = src_rowset->version(); writer_context.segments_overlap = NONOVERLAPPING; @@ -3384,8 +3385,9 @@ Status TabletUpdates::convert_from(const std::shared_ptr& base_tablet, i LOG(INFO) << err_msg_header << "convert_from finish tablet:" << _tablet.tablet_id() << " version:" << this->max_version() << " base tablet:" << base_tablet->tablet_id() << " #pending:" << _pending_commits.size() << " time:" << watch.get_elapse_second() << "s" - << " #column:" << _tablet.tablet_schema().num_columns() << " #rowset:" << src_rowsets.size() - << " #file:" << total_files << " #row:" << total_rows << " bytes:" << total_bytes; + << " #column:" << _tablet.thread_safe_get_tablet_schema()->num_columns() + << " #rowset:" << src_rowsets.size() << " #file:" << total_files << " #row:" << total_rows + << " bytes:" << total_bytes; ; return Status::OK(); } @@ -3393,10 +3395,10 @@ Status TabletUpdates::convert_from(const std::shared_ptr& base_tablet, i Status TabletUpdates::_convert_from_base_rowset(const std::shared_ptr& base_tablet, const ChunkIteratorPtr& seg_iterator, ChunkChanger* chunk_changer, const std::unique_ptr& rowset_writer) { - Schema base_schema = ChunkHelper::convert_schema(base_tablet->tablet_schema()); + Schema base_schema = ChunkHelper::convert_schema(base_tablet->thread_safe_get_tablet_schema()); ChunkPtr base_chunk = ChunkHelper::new_chunk(base_schema, config::vector_chunk_size); - Schema new_schema = ChunkHelper::convert_schema(_tablet.tablet_schema()); + Schema new_schema = ChunkHelper::convert_schema(_tablet.thread_safe_get_tablet_schema()); ChunkPtr new_chunk = ChunkHelper::new_chunk(new_schema, config::vector_chunk_size); std::unique_ptr mem_pool(new MemPool()); @@ -3482,7 +3484,7 @@ Status TabletUpdates::reorder_from(const std::shared_ptr& base_tablet, i std::vector chunk_arr; - Schema base_schema = ChunkHelper::convert_schema(base_tablet->tablet_schema()); + Schema base_schema = ChunkHelper::convert_schema(base_tablet->thread_safe_get_tablet_schema()); ChunkSorter chunk_sorter; OlapReaderStatistics stats; @@ -3510,7 +3512,7 @@ Status TabletUpdates::reorder_from(const std::shared_ptr& base_tablet, i writer_context.partition_id = _tablet.partition_id(); writer_context.tablet_schema_hash = _tablet.schema_hash(); writer_context.rowset_path_prefix = _tablet.schema_hash_path(); - writer_context.tablet_schema = &_tablet.tablet_schema(); + writer_context.tablet_schema = _tablet.thread_safe_get_tablet_schema(); writer_context.rowset_state = VISIBLE; writer_context.version = src_rowset->version(); writer_context.segments_overlap = src_rowset->rowset_meta()->segments_overlap(); @@ -3525,7 +3527,7 @@ Status TabletUpdates::reorder_from(const std::shared_ptr& base_tablet, i ChunkPtr base_chunk = ChunkHelper::new_chunk(base_schema, config::vector_chunk_size); - Schema new_schema = ChunkHelper::convert_schema(_tablet.tablet_schema()); + Schema new_schema = ChunkHelper::convert_schema(_tablet.thread_safe_get_tablet_schema()); for (auto& seg_iterator : seg_iterators) { if (seg_iterator.get() == nullptr) { @@ -3686,8 +3688,9 @@ Status TabletUpdates::reorder_from(const std::shared_ptr& base_tablet, i LOG(INFO) << err_msg_header << "reorder_from finish tablet:" << _tablet.tablet_id() << " version:" << this->max_version() << " base tablet:" << base_tablet->tablet_id() << " #pending:" << _pending_commits.size() << " time:" << watch.get_elapse_second() << "s" - << " #column:" << _tablet.tablet_schema().num_columns() << " #rowset:" << src_rowsets.size() - << " #file:" << total_files << " #row:" << total_rows << " bytes:" << total_bytes; + << " #column:" << _tablet.thread_safe_get_tablet_schema()->num_columns() + << " #rowset:" << src_rowsets.size() << " #file:" << total_files << " #row:" << total_rows + << " bytes:" << total_bytes; return Status::OK(); } @@ -3925,8 +3928,8 @@ Status TabletUpdates::load_snapshot(const SnapshotMeta& snapshot_meta, bool rest if (rowset_meta->tablet_id() != _tablet.tablet_id()) { return Status::InternalError("mismatched tablet id"); } - RETURN_IF_ERROR(RowsetFactory::create_rowset(&_tablet.tablet_schema(), _tablet.schema_hash_path(), - rowset_meta, &rowset)); + RETURN_IF_ERROR(RowsetFactory::create_rowset(_tablet.thread_safe_get_tablet_schema(), + _tablet.schema_hash_path(), rowset_meta, &rowset)); if (rowset->start_version() != rowset->end_version()) { return Status::InternalError("mismatched start and end version"); } @@ -3997,8 +4000,8 @@ Status TabletUpdates::load_snapshot(const SnapshotMeta& snapshot_meta, bool rest std::max(new_next_rowset_id, new_id + std::max(1L, rowset_meta_pb.num_segments())); rowset_meta->set_rowset_seg_id(new_id); RowsetSharedPtr* rowset = &new_rowsets[new_id]; - RETURN_IF_ERROR(RowsetFactory::create_rowset(&_tablet.tablet_schema(), _tablet.schema_hash_path(), - rowset_meta, rowset)); + RETURN_IF_ERROR(RowsetFactory::create_rowset(_tablet.thread_safe_get_tablet_schema(), + _tablet.schema_hash_path(), rowset_meta, rowset)); VLOG(2) << "add a new rowset " << tablet_id << "@" << new_id << "@" << rowset_meta->rowset_id(); } @@ -4244,7 +4247,7 @@ Status TabletUpdates::get_column_values(const std::vector& column_ids, vector>* columns, void* state) { std::vector unique_column_ids; for (unsigned int column_id : column_ids) { - const TabletColumn& tablet_column = _tablet.tablet_schema().column(column_id); + const TabletColumn& tablet_column = _tablet.thread_safe_get_tablet_schema()->column(column_id); unique_column_ids.push_back(tablet_column.unique_id()); } @@ -4265,7 +4268,7 @@ Status TabletUpdates::get_column_values(const std::vector& column_ids, } if (with_default && state == nullptr) { for (auto i = 0; i < column_ids.size(); ++i) { - const TabletColumn& tablet_column = _tablet.tablet_schema().column(column_ids[i]); + const TabletColumn& tablet_column = _tablet.thread_safe_get_tablet_schema()->column(column_ids[i]); if (tablet_column.has_default_value()) { const TypeInfoPtr& type_info = get_type_info(tablet_column); std::unique_ptr default_value_iter = @@ -4300,7 +4303,7 @@ Status TabletUpdates::get_column_values(const std::vector& column_ids, } std::string seg_path = Rowset::segment_file_path(rowset->rowset_path(), rowset->rowset_id(), rssid - iter->first); - auto segment = Segment::open(fs, seg_path, rssid - iter->first, &rowset->schema()); + auto segment = Segment::open(fs, seg_path, rssid - iter->first, rowset->schema()); if (!segment.ok()) { LOG(WARNING) << "Fail to open " << seg_path << ": " << segment.status(); return segment.status(); diff --git a/be/src/storage/task/engine_checksum_task.cpp b/be/src/storage/task/engine_checksum_task.cpp index 5fbfa3198ca90..5b6afdedfb784 100644 --- a/be/src/storage/task/engine_checksum_task.cpp +++ b/be/src/storage/task/engine_checksum_task.cpp @@ -79,11 +79,11 @@ Status EngineChecksumTask::_compute_checksum() { } std::vector return_columns; - const TabletSchema& tablet_schema = tablet->tablet_schema(); + auto tablet_schema = tablet->tablet_schema(); - size_t num_columns = tablet_schema.num_columns(); + size_t num_columns = tablet_schema->num_columns(); for (size_t i = 0; i < num_columns; ++i) { - LogicalType type = tablet_schema.column(i).type(); + LogicalType type = tablet_schema->column(i).type(); // The approximation of FLOAT/DOUBLE in a certain precision range, the binary of byte is not // a fixed value, so these two types are ignored in calculating checksum. // And also HLL/OBJCET/PERCENTILE is too large to calculate the checksum. diff --git a/be/src/storage/task/engine_clone_task.cpp b/be/src/storage/task/engine_clone_task.cpp index bd8cce57f75ff..f2a2a473ecfac 100644 --- a/be/src/storage/task/engine_clone_task.cpp +++ b/be/src/storage/task/engine_clone_task.cpp @@ -912,7 +912,7 @@ Status EngineCloneTask::_clone_full_data(Tablet* tablet, TabletMeta* cloned_tabl // but some rowset is useless, so that remove them here for (auto& rs_meta_ptr : rs_metas_found_in_src) { RowsetSharedPtr rowset_to_remove; - if (auto s = RowsetFactory::create_rowset(&(cloned_tablet_meta->tablet_schema()), tablet->schema_hash_path(), + if (auto s = RowsetFactory::create_rowset(cloned_tablet_meta->tablet_schema_ptr(), tablet->schema_hash_path(), rs_meta_ptr, &rowset_to_remove); !s.ok()) { LOG(WARNING) << "failed to init rowset to remove: " << rs_meta_ptr->rowset_id().to_string(); diff --git a/be/src/storage/update_compaction_state.cpp b/be/src/storage/update_compaction_state.cpp index 76e72a7be176a..b9198521517eb 100644 --- a/be/src/storage/update_compaction_state.cpp +++ b/be/src/storage/update_compaction_state.cpp @@ -66,9 +66,9 @@ Status CompactionState::load_segments(Rowset* rowset, uint32_t segment_id) { static const size_t large_compaction_memory_threshold = 1000000000; Status CompactionState::_load_segments(Rowset* rowset, uint32_t segment_id) { - auto& schema = rowset->schema(); + const auto& schema = rowset->schema(); vector pk_columns; - for (size_t i = 0; i < schema.num_key_columns(); i++) { + for (size_t i = 0; i < schema->num_key_columns(); i++) { pk_columns.push_back(static_cast(i)); } diff --git a/be/src/storage/vertical_compaction_task.cpp b/be/src/storage/vertical_compaction_task.cpp index 428ddfa0d710a..c2c19d9fcae6e 100644 --- a/be/src/storage/vertical_compaction_task.cpp +++ b/be/src/storage/vertical_compaction_task.cpp @@ -62,7 +62,7 @@ Status VerticalCompactionTask::_vertical_compaction_data(Statistics* statistics) _tablet.get(), max_rows_per_segment, _task_info.algorithm, _task_info.output_version, &output_rs_writer)); std::vector> column_groups; - CompactionUtils::split_column_into_groups(_tablet->num_columns(), _tablet->tablet_schema().sort_key_idxes(), + CompactionUtils::split_column_into_groups(_tablet->num_columns(), _tablet->tablet_schema()->sort_key_idxes(), config::vertical_compaction_max_columns_per_group, &column_groups); _task_info.column_group_size = column_groups.size(); @@ -169,6 +169,10 @@ StatusOr VerticalCompactionTask::_calculate_chunk_size_for_column_group total_num_rows += rowset->num_rows(); for (auto& segment : rowset->segments()) { for (uint32_t column_index : column_group) { + if (!segment->is_valid_column(_tablet->tablet_schema()->column(column_index).unique_id())) { + continue; + } + const auto* column_reader = segment->column(column_index); if (column_reader == nullptr) { continue; diff --git a/be/src/tools/meta_tool.cpp b/be/src/tools/meta_tool.cpp index 97d1e6b698212..943ea89ee47b8 100644 --- a/be/src/tools/meta_tool.cpp +++ b/be/src/tools/meta_tool.cpp @@ -517,8 +517,8 @@ void check_meta_consistency(DataDir* data_dir) { tablet_path = starrocks::path_util::join_path_segments(tablet_path, std::to_string(tablet_meta->tablet_id())); tablet_path = starrocks::path_util::join_path_segments(tablet_path, std::to_string(tablet_meta->schema_hash())); - auto& tablet_schema = tablet_meta->tablet_schema(); - const std::vector& columns = tablet_schema.columns(); + auto tablet_schema = tablet_meta->tablet_schema_ptr(); + const std::vector& columns = tablet_schema->columns(); for (const auto& rs : tablet_meta->all_rs_metas()) { for (int64_t seg_id = 0; seg_id < rs->num_segments(); ++seg_id) { @@ -664,7 +664,7 @@ Status SegmentDump::_init() { // open segment size_t footer_length = 16 * 1024 * 1024; - auto segment_res = Segment::open(_fs, _path, 0, _tablet_schema.get(), &footer_length, nullptr); + auto segment_res = Segment::open(_fs, _path, 0, _tablet_schema, &footer_length, nullptr); if (!segment_res.ok()) { std::cout << "open segment failed: " << segment_res.status() << std::endl; return Status::InternalError(""); diff --git a/be/test/runtime/lake_tablets_channel_test.cpp b/be/test/runtime/lake_tablets_channel_test.cpp index f5d400aa1e410..221b1fb89b7f4 100644 --- a/be/test/runtime/lake_tablets_channel_test.cpp +++ b/be/test/runtime/lake_tablets_channel_test.cpp @@ -61,7 +61,7 @@ class LakeTabletsChannelTest : public testing::Test { auto metadata = new_tablet_metadata(10086); _tablet_schema = TabletSchema::create(metadata->schema()); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); // init _open_request _open_request.mutable_id()->set_hi(456789); @@ -217,7 +217,7 @@ class LakeTabletsChannelTest : public testing::Test { auto path = _location_provider->segment_location(tablet_id, filename); std::cerr << path << '\n'; - ASSIGN_OR_ABORT(auto seg, Segment::open(fs, path, 0, _tablet_schema.get())); + ASSIGN_OR_ABORT(auto seg, Segment::open(fs, path, 0, _tablet_schema)); OlapReaderStatistics statistics; SegmentReadOptions opts; diff --git a/be/test/runtime/load_channel_test.cpp b/be/test/runtime/load_channel_test.cpp index b28b07bd0eb60..0168f71666b30 100644 --- a/be/test/runtime/load_channel_test.cpp +++ b/be/test/runtime/load_channel_test.cpp @@ -60,7 +60,7 @@ class LoadChannelTestForLakeTablet : public testing::Test { auto metadata = new_tablet_metadata(10086); _tablet_schema = TabletSchema::create(metadata->schema()); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); // init _open_request _open_request.set_is_lake_tablet(true); @@ -212,7 +212,7 @@ class LoadChannelTestForLakeTablet : public testing::Test { ASSIGN_OR_ABORT(auto fs, FileSystem::CreateSharedFromString(kTestGroupPath)); auto path = _location_provider->segment_location(tablet_id, filename); - ASSIGN_OR_ABORT(auto seg, Segment::open(fs, path, 0, _tablet_schema.get())); + ASSIGN_OR_ABORT(auto seg, Segment::open(fs, path, 0, _tablet_schema)); OlapReaderStatistics statistics; SegmentReadOptions opts; diff --git a/be/test/storage/base_compaction_test.cpp b/be/test/storage/base_compaction_test.cpp index 717f9dab06afc..dd608c6ff34fb 100644 --- a/be/test/storage/base_compaction_test.cpp +++ b/be/test/storage/base_compaction_test.cpp @@ -51,7 +51,7 @@ class BaseCompactionTest : public testing::Test { rowset_writer_context->partition_id = 10; rowset_writer_context->rowset_path_prefix = config::storage_root_path + "/data/0/12345/1111"; rowset_writer_context->rowset_state = VISIBLE; - rowset_writer_context->tablet_schema = _tablet_schema.get(); + rowset_writer_context->tablet_schema = _tablet_schema; rowset_writer_context->version.first = 0; rowset_writer_context->version.second = 1; } @@ -118,7 +118,7 @@ class BaseCompactionTest : public testing::Test { void rowset_writer_add_rows(std::unique_ptr& writer) { std::vector test_data; - auto schema = ChunkHelper::convert_schema(*_tablet_schema); + auto schema = ChunkHelper::convert_schema(_tablet_schema); auto chunk = ChunkHelper::new_chunk(schema, 1024); for (size_t i = 0; i < 1024; ++i) { test_data.push_back("well" + std::to_string(i)); @@ -244,7 +244,7 @@ class BaseCompactionTest : public testing::Test { protected: StorageEngine* _engine = nullptr; StorageEngine* _origin_engine = nullptr; - std::unique_ptr _tablet_schema; + std::shared_ptr _tablet_schema; std::string _schema_hash_path; std::unique_ptr _compaction_mem_tracker; std::unique_ptr _mem_pool; diff --git a/be/test/storage/binlog_manager_test.cpp b/be/test/storage/binlog_manager_test.cpp index e51f526251af9..4a146853b446d 100644 --- a/be/test/storage/binlog_manager_test.cpp +++ b/be/test/storage/binlog_manager_test.cpp @@ -64,7 +64,7 @@ class BinlogManagerTest : public BinlogTestBase { col.set_has_bitmap_index(false); _tablet_schema = std::make_unique(schema_pb); - _schema = ChunkHelper::convert_schema(*_tablet_schema); + _schema = ChunkHelper::convert_schema(_tablet_schema); } void create_rowset(int version, std::vector rows_per_segment, RowsetSharedPtr& rowset) { @@ -78,7 +78,7 @@ class BinlogManagerTest : public BinlogTestBase { writer_context.version = Version(version, 0); writer_context.rowset_path_prefix = _binlog_file_dir; writer_context.rowset_state = VISIBLE; - writer_context.tablet_schema = _tablet_schema.get(); + writer_context.tablet_schema = _tablet_schema; writer_context.writer_type = kHorizontal; std::unique_ptr rowset_writer; @@ -110,7 +110,7 @@ class BinlogManagerTest : public BinlogTestBase { std::vector* metas_for_each_page); int64_t _next_rowset_uid; - std::unique_ptr _tablet_schema; + std::shared_ptr _tablet_schema; Schema _schema; std::shared_ptr _fs; std::string _binlog_file_dir = "binlog_manager_test"; diff --git a/be/test/storage/binlog_reader_test.cpp b/be/test/storage/binlog_reader_test.cpp index 973a0e35dac0d..5cc10e4ca7ad4 100644 --- a/be/test/storage/binlog_reader_test.cpp +++ b/be/test/storage/binlog_reader_test.cpp @@ -128,7 +128,7 @@ void BinlogReaderTest::create_rowset(int32_t* start_key, RowsetInfo& rowset_info writer_context.partition_id = 10; writer_context.rowset_path_prefix = _tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &_tablet->tablet_schema(); + writer_context.tablet_schema = _tablet->tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; diff --git a/be/test/storage/conjunctive_predicates_test.cpp b/be/test/storage/conjunctive_predicates_test.cpp index 60c78f1fa577f..580ccae1e3a6f 100644 --- a/be/test/storage/conjunctive_predicates_test.cpp +++ b/be/test/storage/conjunctive_predicates_test.cpp @@ -377,7 +377,7 @@ TEST_P(ConjunctiveTestFixture, test_parse_conjuncts) { ASSERT_EQ(1, cm.column_value_ranges.count(slot->col_name())); { - PredicateParser pp(*tablet_schema); + PredicateParser pp(tablet_schema); std::unique_ptr predicate(pp.parse_thrift_cond(cm.olap_filters[0])); ASSERT_TRUE(!!predicate); diff --git a/be/test/storage/cumulative_compaction_test.cpp b/be/test/storage/cumulative_compaction_test.cpp index 7ea7f1f924007..94ec80ce9e027 100644 --- a/be/test/storage/cumulative_compaction_test.cpp +++ b/be/test/storage/cumulative_compaction_test.cpp @@ -168,7 +168,7 @@ class CumulativeCompactionTest : public testing::Test { rowset_writer_context->partition_id = 10; rowset_writer_context->rowset_path_prefix = config::storage_root_path + "/data/0/12345/1111"; rowset_writer_context->rowset_state = VISIBLE; - rowset_writer_context->tablet_schema = _tablet_schema.get(); + rowset_writer_context->tablet_schema = _tablet_schema; rowset_writer_context->version.first = version; rowset_writer_context->version.second = version; } @@ -234,7 +234,7 @@ class CumulativeCompactionTest : public testing::Test { void rowset_writer_add_rows(std::unique_ptr& writer) { std::vector test_data; - auto schema = ChunkHelper::convert_schema(*_tablet_schema); + auto schema = ChunkHelper::convert_schema(_tablet_schema); for (size_t j = 0; j < 8; ++j) { auto chunk = ChunkHelper::new_chunk(schema, 128); for (size_t i = 0; i < 128; ++i) { @@ -314,7 +314,7 @@ class CumulativeCompactionTest : public testing::Test { protected: StorageEngine* _engine = nullptr; - std::unique_ptr _tablet_schema; + std::shared_ptr _tablet_schema; std::string _schema_hash_path; std::unique_ptr _metadata_mem_tracker; std::unique_ptr _compaction_mem_tracker; @@ -1004,7 +1004,7 @@ TEST_F(CumulativeCompactionTest, test_issue_20084) { Tablet::create_tablet_from_meta(tablet_meta, starrocks::StorageEngine::instance()->get_stores()[0]); ASSERT_OK(tablet->init()); - std::shared_ptr schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + std::shared_ptr schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); // test reader auto reader = std::make_shared(tablet, Version(0, _version - 1), *schema); ASSERT_OK(reader->prepare()); diff --git a/be/test/storage/default_compaction_policy_test.cpp b/be/test/storage/default_compaction_policy_test.cpp index 16a3d6274cea8..9244cc5c0330a 100644 --- a/be/test/storage/default_compaction_policy_test.cpp +++ b/be/test/storage/default_compaction_policy_test.cpp @@ -125,7 +125,7 @@ class DefaultCompactionPolicyTest : public testing::Test { rowset_writer_context->partition_id = 10; rowset_writer_context->rowset_path_prefix = config::storage_root_path + "/data/0/12345/1111"; rowset_writer_context->rowset_state = VISIBLE; - rowset_writer_context->tablet_schema = _tablet_schema.get(); + rowset_writer_context->tablet_schema = _tablet_schema; rowset_writer_context->version.first = version; rowset_writer_context->version.second = version; } @@ -191,7 +191,7 @@ class DefaultCompactionPolicyTest : public testing::Test { void rowset_writer_add_rows(std::unique_ptr& writer) { std::vector test_data; - auto schema = ChunkHelper::convert_schema(*_tablet_schema); + auto schema = ChunkHelper::convert_schema(_tablet_schema); for (size_t j = 0; j < 8; ++j) { auto chunk = ChunkHelper::new_chunk(schema, 128); for (size_t i = 0; i < 128; ++i) { @@ -282,7 +282,7 @@ class DefaultCompactionPolicyTest : public testing::Test { protected: StorageEngine* _engine = nullptr; - std::unique_ptr _tablet_schema; + std::shared_ptr _tablet_schema; std::string _schema_hash_path; std::unique_ptr _metadata_mem_tracker; std::unique_ptr _compaction_mem_tracker; diff --git a/be/test/storage/delete_handler_test.cpp b/be/test/storage/delete_handler_test.cpp index fb43f334d49d8..ad14569fb5465 100644 --- a/be/test/storage/delete_handler_test.cpp +++ b/be/test/storage/delete_handler_test.cpp @@ -327,7 +327,8 @@ TEST_F(TestDeleteConditionHandler, StoreCondSucceed) { conditions.push_back(condition); DeletePredicatePB del_pred; - success_res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); + success_res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred); ASSERT_EQ(true, success_res.ok()); // Verify that the filter criteria stored in the header are correct @@ -349,8 +350,8 @@ TEST_F(TestDeleteConditionHandler, StoreCondSucceed) { TEST_F(TestDeleteConditionHandler, StoreCondInvalidParameters) { std::vector conditions; DeletePredicatePB del_pred; - Status failed_res = - _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); + Status failed_res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), + conditions, &del_pred); ASSERT_EQ(true, failed_res.is_invalid_argument()); } @@ -364,8 +365,8 @@ TEST_F(TestDeleteConditionHandler, StoreCondNonexistentColumn) { condition.condition_values.emplace_back("2"); conditions.push_back(condition); DeletePredicatePB del_pred; - Status failed_res = - _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); + Status failed_res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), + conditions, &del_pred); ASSERT_TRUE(failed_res.is_invalid_argument()); // 'v' is a value column @@ -376,7 +377,8 @@ TEST_F(TestDeleteConditionHandler, StoreCondNonexistentColumn) { condition.condition_values.emplace_back("5"); conditions.push_back(condition); - failed_res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); + failed_res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred); ASSERT_TRUE(failed_res.is_invalid_argument()); // value column in duplicate model can be deleted; @@ -387,8 +389,8 @@ TEST_F(TestDeleteConditionHandler, StoreCondNonexistentColumn) { condition.condition_values.emplace_back("5"); conditions.push_back(condition); - Status success_res = - _delete_condition_handler.generate_delete_predicate(dup_tablet->tablet_schema(), conditions, &del_pred); + Status success_res = _delete_condition_handler.generate_delete_predicate(dup_tablet->unsafe_tablet_schema_ref(), + conditions, &del_pred); ASSERT_EQ(true, success_res.ok()); } @@ -457,7 +459,8 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { conditions.push_back(condition); DeletePredicatePB del_pred; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred); ASSERT_TRUE(res.ok()); // k5 type is int128 @@ -469,7 +472,8 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { conditions.push_back(condition); DeletePredicatePB del_pred_2; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_2); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_2); ASSERT_TRUE(res.ok()); // k9 type is decimal, precision=6, frac=3 @@ -481,25 +485,29 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { conditions.push_back(condition); DeletePredicatePB del_pred_3; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_3); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_3); ASSERT_EQ(true, res.ok()); conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("2"); DeletePredicatePB del_pred_4; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_4); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_4); ASSERT_TRUE(res.ok()); conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("-2"); DeletePredicatePB del_pred_5; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_5); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_5); ASSERT_TRUE(res.ok()); conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("-2.3"); DeletePredicatePB del_pred_6; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_6); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_6); ASSERT_TRUE(res.ok()); // k10,k11 type is date, datetime @@ -517,7 +525,8 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { conditions.push_back(condition); DeletePredicatePB del_pred_7; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_7); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_7); ASSERT_TRUE(res.ok()); // k12,k13 type is string(64), varchar(64) @@ -535,7 +544,8 @@ TEST_F(TestDeleteConditionHandler2, ValidConditionValue) { conditions.push_back(condition); DeletePredicatePB del_pred_8; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_8); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_8); ASSERT_TRUE(res.ok()); } @@ -552,14 +562,16 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions.push_back(condition); DeletePredicatePB del_pred_1; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_1); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_1); ASSERT_TRUE(res.is_invalid_argument()); // test k1 min, k1 type is int8 conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("-1000"); DeletePredicatePB del_pred_2; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_2); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_2); ASSERT_TRUE(res.is_invalid_argument()); // k2(int16) max value @@ -567,14 +579,16 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].column_name = "k2"; conditions[0].condition_values.emplace_back("32768"); DeletePredicatePB del_pred_3; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_3); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_3); ASSERT_TRUE(res.is_invalid_argument()); // k2(int16) min value conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("-32769"); DeletePredicatePB del_pred_4; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_4); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_4); ASSERT_TRUE(res.is_invalid_argument()); // k3(int32) max @@ -582,14 +596,16 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].column_name = "k3"; conditions[0].condition_values.emplace_back("2147483648"); DeletePredicatePB del_pred_5; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_5); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_5); ASSERT_TRUE(res.is_invalid_argument()); // k3(int32) min value conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("-2147483649"); DeletePredicatePB del_pred_6; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_6); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_6); ASSERT_TRUE(res.is_invalid_argument()); // k4(int64) max @@ -597,14 +613,16 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].column_name = "k4"; conditions[0].condition_values.emplace_back("9223372036854775808"); DeletePredicatePB del_pred_7; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_7); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_7); ASSERT_TRUE(res.is_invalid_argument()); // k4(int64) min conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("-9223372036854775809"); DeletePredicatePB del_pred_8; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_8); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_8); ASSERT_TRUE(res.is_invalid_argument()); // k5(int128) max @@ -612,14 +630,16 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].column_name = "k5"; conditions[0].condition_values.emplace_back("170141183460469231731687303715884105728"); DeletePredicatePB del_pred_9; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_9); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_9); ASSERT_TRUE(res.is_invalid_argument()); // k5(int128) min conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("-170141183460469231731687303715884105729"); DeletePredicatePB del_pred_10; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_10); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_10); ASSERT_TRUE(res.is_invalid_argument()); // k9 integer overflow, type is decimal, precision=6, frac=3 @@ -627,21 +647,24 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].column_name = "k9"; conditions[0].condition_values.emplace_back("12347876.5"); DeletePredicatePB del_pred_11; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_11); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_11); ASSERT_TRUE(res.is_invalid_argument()); // k9 scale overflow, type is decimal, precision=6, frac=3 conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("1.2345678"); DeletePredicatePB del_pred_12; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_12); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_12); ASSERT_TRUE(res.is_invalid_argument()); // k9 has point conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("1."); DeletePredicatePB del_pred_13; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_13); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_13); ASSERT_TRUE(res.is_invalid_argument()); // invalid date @@ -649,20 +672,23 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].column_name = "k10"; conditions[0].condition_values.emplace_back("20130101"); DeletePredicatePB del_pred_14; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_14); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_14); ASSERT_TRUE(res.is_invalid_argument()); conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("2013-64-01"); DeletePredicatePB del_pred_15; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_15); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_15); ASSERT_TRUE(res.is_invalid_argument()); conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("2013-01-40"); DeletePredicatePB del_pred_16; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_16); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_16); ASSERT_TRUE(res.is_invalid_argument()); // invalid datetime @@ -670,37 +696,43 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { conditions[0].column_name = "k11"; conditions[0].condition_values.emplace_back("20130101 00:00:00"); DeletePredicatePB del_pred_17; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_17); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_17); ASSERT_TRUE(res.is_invalid_argument()); conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("2013-64-01 00:00:00"); DeletePredicatePB del_pred_18; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_18); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_18); ASSERT_TRUE(res.is_invalid_argument()); conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("2013-01-40 00:00:00"); DeletePredicatePB del_pred_19; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_19); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_19); ASSERT_TRUE(res.is_invalid_argument()); conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("2013-01-01 24:00:00"); DeletePredicatePB del_pred_20; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_20); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_20); ASSERT_TRUE(res.is_invalid_argument()); conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("2013-01-01 00:60:00"); DeletePredicatePB del_pred_21; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_21); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_21); ASSERT_TRUE(res.is_invalid_argument()); conditions[0].condition_values.clear(); conditions[0].condition_values.emplace_back("2013-01-01 00:00:60"); DeletePredicatePB del_pred_22; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_22); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_22); ASSERT_TRUE(res.is_invalid_argument()); // too long varchar @@ -711,7 +743,8 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { "FhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYW" "FhYWFhYWFhYWFhYWFhYWFhYWFhYWE=;k13=YWFhYQ=="); DeletePredicatePB del_pred_23; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_23); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_23); ASSERT_TRUE(res.is_invalid_argument()); conditions[0].condition_values.clear(); @@ -721,7 +754,8 @@ TEST_F(TestDeleteConditionHandler2, InvalidConditionValue) { "FhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYW" "FhYWFhYWFhYWFhYWFhYWFhYWFhYWE=;k13=YWFhYQ=="); DeletePredicatePB del_pred_24; - res = _delete_condition_handler.generate_delete_predicate(tablet->tablet_schema(), conditions, &del_pred_24); + res = _delete_condition_handler.generate_delete_predicate(tablet->unsafe_tablet_schema_ref(), conditions, + &del_pred_24); ASSERT_TRUE(res.is_invalid_argument()); } diff --git a/be/test/storage/get_use_pk_index_test.cpp b/be/test/storage/get_use_pk_index_test.cpp index a6c43d3918859..663d9a91a8920 100644 --- a/be/test/storage/get_use_pk_index_test.cpp +++ b/be/test/storage/get_use_pk_index_test.cpp @@ -55,7 +55,7 @@ class GetUsePkIndexTest : public testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = overlap; @@ -209,7 +209,7 @@ class GetUsePkIndexTest : public testing::Test { } void read_using_pk_index(int64_t key, int64_t version, bool multi_column_pk, bool expect_exist) { - const Schema& schema = *_tablet->tablet_schema().schema(); + const Schema& schema = *_tablet->tablet_schema()->schema(); TabletReader reader(_tablet, Version(0, version), schema); TabletReaderParams params; params.is_pipeline = true; diff --git a/be/test/storage/lake/async_delta_writer_test.cpp b/be/test/storage/lake/async_delta_writer_test.cpp index 71fb20395fc08..00d0b4f12e54c 100644 --- a/be/test/storage/lake/async_delta_writer_test.cpp +++ b/be/test/storage/lake/async_delta_writer_test.cpp @@ -77,7 +77,7 @@ class LakeAsyncDeltaWriterTest : public TestBase { } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } protected: @@ -210,7 +210,7 @@ TEST_F(LakeAsyncDeltaWriterTest, test_write) { ASSIGN_OR_ABORT(auto fs, FileSystem::CreateSharedFromString(kTestDirectory)); auto path0 = _tablet_mgr->segment_location(tablet_id, txnlog->op_write().rowset().segments(0)); - ASSIGN_OR_ABORT(auto seg0, Segment::open(fs, path0, 0, _tablet_schema.get())); + ASSIGN_OR_ABORT(auto seg0, Segment::open(fs, path0, 0, _tablet_schema)); OlapReaderStatistics statistics; SegmentReadOptions opts; @@ -306,7 +306,7 @@ TEST_F(LakeAsyncDeltaWriterTest, test_write_concurrently) { ASSIGN_OR_ABORT(auto fs, FileSystem::CreateSharedFromString(kTestDirectory)); auto path0 = _tablet_mgr->segment_location(tablet_id, txnlog->op_write().rowset().segments(0)); - ASSIGN_OR_ABORT(auto seg0, Segment::open(fs, path0, 0, _tablet_schema.get())); + ASSIGN_OR_ABORT(auto seg0, Segment::open(fs, path0, 0, _tablet_schema)); OlapReaderStatistics statistics; SegmentReadOptions opts; diff --git a/be/test/storage/lake/auto_increment_partial_update_test.cpp b/be/test/storage/lake/auto_increment_partial_update_test.cpp index 6a01efe333d2a..ff2c85ddcc583 100644 --- a/be/test/storage/lake/auto_increment_partial_update_test.cpp +++ b/be/test/storage/lake/auto_increment_partial_update_test.cpp @@ -75,7 +75,7 @@ class AutoIncrementPartialUpdateTest : public TestBase { _referenced_column_ids.push_back(0); _referenced_column_ids.push_back(1); _partial_tablet_schema = TabletSchema::create(*schema); - _partial_schema = std::make_shared(ChunkHelper::convert_schema(*_partial_tablet_schema)); + _partial_schema = std::make_shared(ChunkHelper::convert_schema(_partial_tablet_schema)); auto c2 = schema->add_column(); { @@ -88,7 +88,7 @@ class AutoIncrementPartialUpdateTest : public TestBase { } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } void SetUp() override { diff --git a/be/test/storage/lake/compaction_task_test.cpp b/be/test/storage/lake/compaction_task_test.cpp index b993027c2c658..2f8cd764f0d39 100644 --- a/be/test/storage/lake/compaction_task_test.cpp +++ b/be/test/storage/lake/compaction_task_test.cpp @@ -114,7 +114,7 @@ class LakeDuplicateKeyCompactionTest : public LakeCompactionTest { } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } protected: @@ -273,7 +273,7 @@ class LakeDuplicateKeyOverlapSegmentsCompactionTest : public LakeCompactionTest } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } protected: @@ -448,7 +448,7 @@ class LakeUniqueKeyCompactionTest : public LakeCompactionTest { } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } protected: @@ -590,7 +590,7 @@ class LakeUniqueKeyCompactionWithDeleteTest : public LakeCompactionTest { } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } protected: diff --git a/be/test/storage/lake/condition_update_test.cpp b/be/test/storage/lake/condition_update_test.cpp index 98b5d7e3418bc..bcb52ca4b7dda 100644 --- a/be/test/storage/lake/condition_update_test.cpp +++ b/be/test/storage/lake/condition_update_test.cpp @@ -75,7 +75,7 @@ class ConditionUpdateTest : public TestBase { _referenced_column_ids.push_back(0); _referenced_column_ids.push_back(1); _partial_tablet_schema = TabletSchema::create(*schema); - _partial_schema = std::make_shared(ChunkHelper::convert_schema(*_partial_tablet_schema)); + _partial_schema = std::make_shared(ChunkHelper::convert_schema(_partial_tablet_schema)); auto c2 = schema->add_column(); { @@ -88,7 +88,7 @@ class ConditionUpdateTest : public TestBase { } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } void SetUp() override { diff --git a/be/test/storage/lake/delta_writer_test.cpp b/be/test/storage/lake/delta_writer_test.cpp index 57297586dcfeb..8458cc8c8dac3 100644 --- a/be/test/storage/lake/delta_writer_test.cpp +++ b/be/test/storage/lake/delta_writer_test.cpp @@ -77,7 +77,7 @@ class LakeDeltaWriterTest : public TestBase { } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } protected: @@ -172,8 +172,8 @@ TEST_F(LakeDeltaWriterTest, test_write) { auto path0 = _tablet_mgr->segment_location(tablet_id, txnlog->op_write().rowset().segments(0)); auto path1 = _tablet_mgr->segment_location(tablet_id, txnlog->op_write().rowset().segments(1)); - ASSIGN_OR_ABORT(auto seg0, Segment::open(fs, path0, 0, _tablet_schema.get())); - ASSIGN_OR_ABORT(auto seg1, Segment::open(fs, path1, 1, _tablet_schema.get())); + ASSIGN_OR_ABORT(auto seg0, Segment::open(fs, path0, 0, _tablet_schema)); + ASSIGN_OR_ABORT(auto seg1, Segment::open(fs, path1, 1, _tablet_schema)); OlapReaderStatistics statistics; SegmentReadOptions opts; diff --git a/be/test/storage/lake/partial_update_test.cpp b/be/test/storage/lake/partial_update_test.cpp index 5a0aa6667f9fb..712a19b7c6e7f 100644 --- a/be/test/storage/lake/partial_update_test.cpp +++ b/be/test/storage/lake/partial_update_test.cpp @@ -74,7 +74,7 @@ class PartialUpdateTest : public TestBase { _referenced_column_ids.push_back(0); _referenced_column_ids.push_back(1); _partial_tablet_schema = TabletSchema::create(*schema); - _partial_schema = std::make_shared(ChunkHelper::convert_schema(*_partial_tablet_schema)); + _partial_schema = std::make_shared(ChunkHelper::convert_schema(_partial_tablet_schema)); auto c2 = schema->add_column(); { @@ -87,7 +87,7 @@ class PartialUpdateTest : public TestBase { } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } void SetUp() override { diff --git a/be/test/storage/lake/primary_key_compaction_task_test.cpp b/be/test/storage/lake/primary_key_compaction_task_test.cpp index d7f7db37d5936..0646f2f73ea8c 100644 --- a/be/test/storage/lake/primary_key_compaction_task_test.cpp +++ b/be/test/storage/lake/primary_key_compaction_task_test.cpp @@ -81,7 +81,7 @@ class LakePrimaryKeyCompactionTest : public TestBase, public testing::WithParamI } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } protected: diff --git a/be/test/storage/lake/primary_key_publish_test.cpp b/be/test/storage/lake/primary_key_publish_test.cpp index c18bc74004990..a99b5a2134d92 100644 --- a/be/test/storage/lake/primary_key_publish_test.cpp +++ b/be/test/storage/lake/primary_key_publish_test.cpp @@ -80,7 +80,7 @@ class LakePrimaryKeyPublishTest : public TestBase { } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } void SetUp() override { diff --git a/be/test/storage/lake/rowset_test.cpp b/be/test/storage/lake/rowset_test.cpp index 77739e25f4b96..27b07db4a2fad 100644 --- a/be/test/storage/lake/rowset_test.cpp +++ b/be/test/storage/lake/rowset_test.cpp @@ -67,7 +67,7 @@ class LakeRowsetTest : public TestBase { } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } void SetUp() override { diff --git a/be/test/storage/lake/schema_change_test.cpp b/be/test/storage/lake/schema_change_test.cpp index 3d1d87ea6ea59..586c74846ceca 100644 --- a/be/test/storage/lake/schema_change_test.cpp +++ b/be/test/storage/lake/schema_change_test.cpp @@ -148,7 +148,7 @@ class SchemaChangeAddColumnTest : public SchemaChangeTest { } _base_tablet_schema = TabletSchema::create(*base_schema); - _base_schema = std::make_shared(ChunkHelper::convert_schema(*_base_tablet_schema)); + _base_schema = std::make_shared(ChunkHelper::convert_schema(_base_tablet_schema)); // new tablet _new_tablet_metadata = std::make_shared(); @@ -203,7 +203,7 @@ class SchemaChangeAddColumnTest : public SchemaChangeTest { } _new_tablet_schema = TabletSchema::create(*new_schema); - _new_schema = std::make_shared(ChunkHelper::convert_schema(*_new_tablet_schema)); + _new_schema = std::make_shared(ChunkHelper::convert_schema(_new_tablet_schema)); } protected: @@ -406,7 +406,7 @@ class SchemaChangeModifyColumnTypeTest : public SchemaChangeTest { } _base_tablet_schema = TabletSchema::create(*base_schema); - _base_schema = std::make_shared(ChunkHelper::convert_schema(*_base_tablet_schema)); + _base_schema = std::make_shared(ChunkHelper::convert_schema(_base_tablet_schema)); // new tablet _new_tablet_metadata = std::make_shared(); @@ -446,7 +446,7 @@ class SchemaChangeModifyColumnTypeTest : public SchemaChangeTest { } _new_tablet_schema = TabletSchema::create(*new_schema); - _new_schema = std::make_shared(ChunkHelper::convert_schema(*_new_tablet_schema)); + _new_schema = std::make_shared(ChunkHelper::convert_schema(_new_tablet_schema)); } protected: @@ -652,7 +652,7 @@ class SchemaChangeModifyColumnOrderTest : public SchemaChangeTest { } _base_tablet_schema = TabletSchema::create(*base_schema); - _base_schema = std::make_shared(ChunkHelper::convert_schema(*_base_tablet_schema)); + _base_schema = std::make_shared(ChunkHelper::convert_schema(_base_tablet_schema)); // new tablet _new_tablet_metadata = std::make_shared(); @@ -702,7 +702,7 @@ class SchemaChangeModifyColumnOrderTest : public SchemaChangeTest { } _new_tablet_schema = TabletSchema::create(*new_schema); - _new_schema = std::make_shared(ChunkHelper::convert_schema(*_new_tablet_schema)); + _new_schema = std::make_shared(ChunkHelper::convert_schema(_new_tablet_schema)); } protected: diff --git a/be/test/storage/lake/tablet_reader_test.cpp b/be/test/storage/lake/tablet_reader_test.cpp index be30331a1e229..260490c2172ae 100644 --- a/be/test/storage/lake/tablet_reader_test.cpp +++ b/be/test/storage/lake/tablet_reader_test.cpp @@ -71,7 +71,7 @@ class LakeDuplicateTabletReaderTest : public TestBase { } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } void SetUp() override { @@ -210,7 +210,7 @@ class LakeAggregateTabletReaderTest : public TestBase { } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } void SetUp() override { @@ -372,7 +372,7 @@ class LakeDuplicateTabletReaderWithDeleteTest : public TestBase { } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } void SetUp() override { @@ -537,7 +537,7 @@ class LakeDuplicateTabletReaderWithDeleteNotInOneValueTest : public TestBase { } _tablet_schema = TabletSchema::create(*schema); - _schema = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } void SetUp() override { diff --git a/be/test/storage/lake/tablet_writer_test.cpp b/be/test/storage/lake/tablet_writer_test.cpp index ecfcd869acb6d..aef3f9262b29b 100644 --- a/be/test/storage/lake/tablet_writer_test.cpp +++ b/be/test/storage/lake/tablet_writer_test.cpp @@ -71,7 +71,7 @@ class LakeTabletWriterTest : public TestBase, testing::WithParamInterface(ChunkHelper::convert_schema(*_tablet_schema)); + _schema = std::make_shared(ChunkHelper::convert_schema(_tablet_schema)); } void SetUp() override { @@ -134,9 +134,9 @@ TEST_P(LakeTabletWriterTest, test_write_success) { ASSIGN_OR_ABORT(auto fs, FileSystem::CreateSharedFromString(kTestDirectory)); ASSIGN_OR_ABORT(auto seg0, Segment::open(fs, _tablet_mgr->segment_location(_tablet_metadata->id(), files[0]), 0, - _tablet_schema.get())); + _tablet_schema)); ASSIGN_OR_ABORT(auto seg1, Segment::open(fs, _tablet_mgr->segment_location(_tablet_metadata->id(), files[1]), 1, - _tablet_schema.get())); + _tablet_schema)); OlapReaderStatistics statistics; SegmentReadOptions opts; @@ -183,8 +183,8 @@ TEST_P(LakeTabletWriterTest, test_vertical_write_success) { c2->append_numbers(k1.data(), k1.size() * sizeof(int)); c3->append_numbers(v1.data(), v1.size() * sizeof(int)); - auto schema0 = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema, {0})); - auto schema1 = std::make_shared(ChunkHelper::convert_schema(*_tablet_schema, {1})); + auto schema0 = std::make_shared(ChunkHelper::convert_schema(_tablet_schema, {0})); + auto schema1 = std::make_shared(ChunkHelper::convert_schema(_tablet_schema, {1})); Chunk c0_chunk({c0}, schema0); Chunk c1_chunk({c1}, schema1); @@ -220,9 +220,9 @@ TEST_P(LakeTabletWriterTest, test_vertical_write_success) { ASSIGN_OR_ABORT(auto fs, FileSystem::CreateSharedFromString(kTestDirectory)); ASSIGN_OR_ABORT(auto seg0, Segment::open(fs, _tablet_mgr->segment_location(_tablet_metadata->id(), files[0]), 0, - _tablet_schema.get())); + _tablet_schema)); ASSIGN_OR_ABORT(auto seg1, Segment::open(fs, _tablet_mgr->segment_location(_tablet_metadata->id(), files[1]), 1, - _tablet_schema.get())); + _tablet_schema)); OlapReaderStatistics statistics; SegmentReadOptions opts; diff --git a/be/test/storage/memtable_flush_executor_test.cpp b/be/test/storage/memtable_flush_executor_test.cpp index 17edcbcd44608..b4d2fd497bebf 100644 --- a/be/test/storage/memtable_flush_executor_test.cpp +++ b/be/test/storage/memtable_flush_executor_test.cpp @@ -215,12 +215,12 @@ class MemTableFlushExecutorTest : public ::testing::Test { writer_context.partition_id = 10; writer_context.rowset_path_prefix = _root_path; writer_context.rowset_state = VISIBLE; - writer_context.tablet_schema = _schema.get(); + writer_context.tablet_schema = _schema; writer_context.version.first = 10; writer_context.version.second = 10; ASSERT_TRUE(RowsetFactory::create_rowset_writer(writer_context, &_writer).ok()); _mem_table_sink = std::make_unique(_writer.get()); - _vectorized_schema = MemTable::convert_schema(_schema.get(), _slots); + _vectorized_schema = MemTable::convert_schema(_schema, _slots); } void TearDown() override { diff --git a/be/test/storage/memtable_test.cpp b/be/test/storage/memtable_test.cpp index c868fb5754e8d..6939433b1daac 100644 --- a/be/test/storage/memtable_test.cpp +++ b/be/test/storage/memtable_test.cpp @@ -218,12 +218,12 @@ class MemTableTest : public ::testing::Test { writer_context.partition_id = 10; writer_context.rowset_path_prefix = _root_path; writer_context.rowset_state = VISIBLE; - writer_context.tablet_schema = _schema.get(); + writer_context.tablet_schema = _schema; writer_context.version.first = 10; writer_context.version.second = 10; ASSERT_TRUE(RowsetFactory::create_rowset_writer(writer_context, &_writer).ok()); _mem_table_sink = std::make_unique(_writer.get()); - _vectorized_schema = std::move(MemTable::convert_schema(_schema.get(), _slots)); + _vectorized_schema = std::move(MemTable::convert_schema(_schema, _slots)); _mem_table = std::make_unique(1, &_vectorized_schema, _slots, _mem_table_sink.get(), _mem_tracker.get()); } @@ -408,7 +408,7 @@ TEST_F(MemTableTest, testPrimaryKeysNullableSortKey) { expected_chunk->get_column_by_index(2)->append_datum(Datum(static_cast(2 * i + 1))); } - Schema read_schema = ChunkHelper::convert_schema(*tablet_schema); + Schema read_schema = ChunkHelper::convert_schema(tablet_schema); OlapReaderStatistics stats; RowsetReadOptions rs_opts; rs_opts.sorted = false; diff --git a/be/test/storage/persistent_index_test.cpp b/be/test/storage/persistent_index_test.cpp index cc10df32ed905..bb2ca2962b910 100644 --- a/be/test/storage/persistent_index_test.cpp +++ b/be/test/storage/persistent_index_test.cpp @@ -1297,7 +1297,7 @@ RowsetSharedPtr create_rowset(const TabletSharedPtr& tablet, const vectorschema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; diff --git a/be/test/storage/publish_version_manager_test.cpp b/be/test/storage/publish_version_manager_test.cpp index d38600982dc9d..9f23978ec7e1a 100644 --- a/be/test/storage/publish_version_manager_test.cpp +++ b/be/test/storage/publish_version_manager_test.cpp @@ -106,7 +106,7 @@ class PublishVersionManagerTest : public testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; diff --git a/be/test/storage/publish_version_task_test.cpp b/be/test/storage/publish_version_task_test.cpp index 32036dddc55f8..393bfa5951714 100644 --- a/be/test/storage/publish_version_task_test.cpp +++ b/be/test/storage/publish_version_task_test.cpp @@ -70,11 +70,11 @@ class PublishVersionTaskTest : public testing::Test { TabletManager* tablet_manager = starrocks::StorageEngine::instance()->tablet_manager(); TabletSharedPtr tablet = tablet_manager->get_tablet(12345); ASSERT_TRUE(tablet != nullptr); - const TabletSchema& tablet_schema = tablet->tablet_schema(); + const TabletSchemaCSPtr& tablet_schema = tablet->tablet_schema(); // create rowset RowsetWriterContext rowset_writer_context; - create_rowset_writer_context(&rowset_writer_context, tablet->schema_hash_path(), &tablet_schema); + create_rowset_writer_context(&rowset_writer_context, tablet->schema_hash_path(), tablet_schema); std::unique_ptr rowset_writer; ASSERT_TRUE(RowsetFactory::create_rowset_writer(rowset_writer_context, &rowset_writer).ok()); @@ -125,7 +125,8 @@ class PublishVersionTaskTest : public testing::Test { } static void create_rowset_writer_context(RowsetWriterContext* rowset_writer_context, - const std::string& schema_hash_path, const TabletSchema* tablet_schema) { + const std::string& schema_hash_path, + const TabletSchemaCSPtr& tablet_schema) { RowsetId rowset_id; rowset_id.init(10000); rowset_writer_context->rowset_id = rowset_id; @@ -139,7 +140,7 @@ class PublishVersionTaskTest : public testing::Test { rowset_writer_context->version.second = 2; } - static void rowset_writer_add_rows(std::unique_ptr& writer, const TabletSchema& tablet_schema) { + static void rowset_writer_add_rows(std::unique_ptr& writer, const TabletSchemaCSPtr& tablet_schema) { std::vector test_data; auto schema = ChunkHelper::convert_schema(tablet_schema); auto chunk = ChunkHelper::new_chunk(schema, 1024); diff --git a/be/test/storage/rowset/column_reader_writer_test.cpp b/be/test/storage/rowset/column_reader_writer_test.cpp index 4c3eaa92a016e..da0e705f396da 100644 --- a/be/test/storage/rowset/column_reader_writer_test.cpp +++ b/be/test/storage/rowset/column_reader_writer_test.cpp @@ -99,7 +99,7 @@ class ColumnReaderWriterTest : public testing::Test { void TearDown() override {} std::shared_ptr create_dummy_segment(const std::shared_ptr& fs, const std::string& fname) { - return std::make_shared(Segment::private_type(0), fs, fname, 1, _dummy_segment_schema.get()); + return std::make_shared(Segment::private_type(0), fs, fname, 1, _dummy_segment_schema); } template diff --git a/be/test/storage/rowset/map_column_rw_test.cpp b/be/test/storage/rowset/map_column_rw_test.cpp index 9f740507964cf..c8494ee1f4cd9 100644 --- a/be/test/storage/rowset/map_column_rw_test.cpp +++ b/be/test/storage/rowset/map_column_rw_test.cpp @@ -48,7 +48,7 @@ class MapColumnRWTest : public testing::Test { void TearDown() override {} std::shared_ptr create_dummy_segment(const std::shared_ptr& fs, const std::string& fname) { - return std::make_shared(Segment::private_type(0), fs, fname, 1, _dummy_segment_schema.get()); + return std::make_shared(Segment::private_type(0), fs, fname, 1, _dummy_segment_schema); } void test_int_map() { diff --git a/be/test/storage/rowset/rowset_test.cpp b/be/test/storage/rowset/rowset_test.cpp index 1ad8aa9958d4e..e28df943eddde 100644 --- a/be/test/storage/rowset/rowset_test.cpp +++ b/be/test/storage/rowset/rowset_test.cpp @@ -204,7 +204,7 @@ class RowsetTest : public testing::Test { return StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id, false); } - void create_rowset_writer_context(int64_t tablet_id, const TabletSchema* tablet_schema, + void create_rowset_writer_context(int64_t tablet_id, const TabletSchemaCSPtr& tablet_schema, RowsetWriterContext* rowset_writer_context) { RowsetId rowset_id; rowset_id.init(10000); @@ -231,7 +231,7 @@ class RowsetTest : public testing::Test { rowset_writer_context->rowset_path_prefix = config::storage_root_path + "/data/rowset_test"; rowset_writer_context->rowset_state = VISIBLE; rowset_writer_context->partial_update_tablet_schema = partial_schema; - rowset_writer_context->tablet_schema = partial_schema.get(); + rowset_writer_context->tablet_schema = partial_schema; rowset_writer_context->referenced_column_ids = column_indexes; rowset_writer_context->version.first = 0; rowset_writer_context->version.second = 0; @@ -267,7 +267,7 @@ void RowsetTest::test_final_merge(bool has_merge_condition = false) { RowsetSharedPtr rowset; const uint32_t rows_per_segment = 1024; RowsetWriterContext writer_context; - create_rowset_writer_context(12421, &tablet->tablet_schema(), &writer_context); + create_rowset_writer_context(12421, tablet->tablet_schema(), &writer_context); writer_context.segments_overlap = OVERLAP_UNKNOWN; if (has_merge_condition) { writer_context.merge_condition = "v1"; @@ -334,7 +334,7 @@ void RowsetTest::test_final_merge(bool has_merge_condition = false) { seg_options.stats = &_stats; std::string segment_file = Rowset::segment_file_path(writer_context.rowset_path_prefix, writer_context.rowset_id, seg_id); - auto segment = *Segment::open(seg_options.fs, segment_file, 0, &tablet->tablet_schema()); + auto segment = *Segment::open(seg_options.fs, segment_file, 0, tablet->tablet_schema()); ASSERT_NE(segment->num_rows(), 0); auto res = segment->new_iterator(schema, seg_options); ASSERT_FALSE(res.status().is_end_of_file() || !res.ok() || res.value() == nullptr); @@ -429,7 +429,7 @@ TEST_F(RowsetTest, FinalMergeVerticalTest) { const uint32_t rows_per_segment = 1024; config::vertical_compaction_max_columns_per_group = 1; RowsetWriterContext writer_context; - create_rowset_writer_context(12345, &tablet->tablet_schema(), &writer_context); + create_rowset_writer_context(12345, tablet->tablet_schema(), &writer_context); writer_context.segments_overlap = OVERLAP_UNKNOWN; std::unique_ptr rowset_writer; @@ -494,7 +494,7 @@ TEST_F(RowsetTest, FinalMergeVerticalTest) { std::string segment_file = Rowset::segment_file_path(writer_context.rowset_path_prefix, writer_context.rowset_id, seg_id); - auto segment = *Segment::open(seg_options.fs, segment_file, 0, &tablet->tablet_schema()); + auto segment = *Segment::open(seg_options.fs, segment_file, 0, tablet->tablet_schema()); ASSERT_NE(segment->num_rows(), 0); auto res = segment->new_iterator(schema, seg_options); @@ -618,7 +618,7 @@ static ssize_t read_and_compare(const ChunkIteratorPtr& iter, int64_t nkeys) { static ssize_t read_tablet_and_compare(const TabletSharedPtr& tablet, const std::shared_ptr& partial_schema, int64_t version, int64_t nkeys) { - Schema schema = ChunkHelper::convert_schema(*partial_schema); + Schema schema = ChunkHelper::convert_schema(partial_schema); TabletReader reader(tablet, Version(0, version), schema); auto iter = create_tablet_iterator(reader, schema); if (iter == nullptr) { @@ -641,7 +641,7 @@ TEST_F(RowsetTest, FinalMergeVerticalPartialTest) { std::unique_ptr rowset_writer; ASSERT_TRUE(RowsetFactory::create_rowset_writer(writer_context, &rowset_writer).ok()); - auto schema = ChunkHelper::convert_schema(*partial_schema); + auto schema = ChunkHelper::convert_schema(partial_schema); { auto chunk = ChunkHelper::new_chunk(schema, config::vector_chunk_size); @@ -683,7 +683,7 @@ TEST_F(RowsetTest, FinalMergeVerticalPartialTest) { } auto rowset = rowset_writer->build().value(); - rowset->set_schema(&tablet->tablet_schema()); + rowset->set_schema(tablet->tablet_schema()); ASSERT_TRUE(rowset != nullptr); ASSERT_EQ(3, rowset->rowset_meta()->num_segments()); ASSERT_EQ(rows_per_segment * 3, rowset->rowset_meta()->num_rows()); @@ -697,7 +697,7 @@ TEST_F(RowsetTest, VerticalWriteTest) { auto tablet_schema = TabletSchemaHelper::create_tablet_schema(); RowsetWriterContext writer_context; - create_rowset_writer_context(12345, tablet_schema.get(), &writer_context); + create_rowset_writer_context(12345, tablet_schema, &writer_context); writer_context.max_rows_per_segment = 5000; writer_context.writer_type = kVertical; @@ -710,7 +710,7 @@ TEST_F(RowsetTest, VerticalWriteTest) { { // k1 k2 std::vector column_indexes{0, 1}; - auto schema = ChunkHelper::convert_schema(*tablet_schema, column_indexes); + auto schema = ChunkHelper::convert_schema(tablet_schema, column_indexes); auto chunk = ChunkHelper::new_chunk(schema, chunk_size); for (auto i = 0; i < num_rows % chunk_size; ++i) { chunk->reset(); @@ -727,7 +727,7 @@ TEST_F(RowsetTest, VerticalWriteTest) { { // v1 std::vector column_indexes{2}; - auto schema = ChunkHelper::convert_schema(*tablet_schema, column_indexes); + auto schema = ChunkHelper::convert_schema(tablet_schema, column_indexes); auto chunk = ChunkHelper::new_chunk(schema, chunk_size); for (auto i = 0; i < num_rows % chunk_size; ++i) { chunk->reset(); @@ -751,7 +751,7 @@ TEST_F(RowsetTest, VerticalWriteTest) { rs_opts.sorted = true; rs_opts.version = 0; rs_opts.stats = &_stats; - auto schema = ChunkHelper::convert_schema(*tablet_schema); + auto schema = ChunkHelper::convert_schema(tablet_schema); auto res = rowset->new_iterator(schema, rs_opts); ASSERT_FALSE(res.status().is_end_of_file() || !res.ok() || res.value() == nullptr); @@ -779,7 +779,7 @@ TEST_F(RowsetTest, SegmentWriteTest) { auto tablet_schema = TabletSchemaHelper::create_tablet_schema(); RowsetWriterContext writer_context; - create_rowset_writer_context(12345, tablet_schema.get(), &writer_context); + create_rowset_writer_context(12345, tablet_schema, &writer_context); writer_context.writer_type = kHorizontal; std::unique_ptr rowset_writer; @@ -792,7 +792,7 @@ TEST_F(RowsetTest, SegmentWriteTest) { { // k1 k2 v std::vector column_indexes{0, 1, 2}; - auto schema = ChunkHelper::convert_schema(*tablet_schema, column_indexes); + auto schema = ChunkHelper::convert_schema(tablet_schema, column_indexes); auto chunk = ChunkHelper::new_chunk(schema, chunk_size); for (auto i = 0; i < num_rows / chunk_size + 1; ++i) { chunk->reset(); @@ -817,7 +817,7 @@ TEST_F(RowsetTest, SegmentWriteTest) { rs_opts.sorted = true; rs_opts.version = 0; rs_opts.stats = &_stats; - auto schema = ChunkHelper::convert_schema(*tablet_schema); + auto schema = ChunkHelper::convert_schema(tablet_schema); auto res = rowset->new_iterator(schema, rs_opts); ASSERT_FALSE(res.status().is_end_of_file() || !res.ok() || res.value() == nullptr); @@ -872,7 +872,7 @@ TEST_F(RowsetTest, SegmentWriteTest) { rs_opts.sorted = true; rs_opts.version = 0; rs_opts.stats = &_stats; - auto schema = ChunkHelper::convert_schema(*tablet_schema); + auto schema = ChunkHelper::convert_schema(tablet_schema); auto res = rowset->new_iterator(schema, rs_opts); ASSERT_FALSE(res.status().is_end_of_file() || !res.ok() || res.value() == nullptr); @@ -898,11 +898,11 @@ TEST_F(RowsetTest, SegmentWriteTest) { } TEST_F(RowsetTest, SegmentRewriterAutoIncrementTest) { - std::unique_ptr partial_tablet_schema = TabletSchemaHelper::create_tablet_schema( + std::shared_ptr partial_tablet_schema = TabletSchemaHelper::create_tablet_schema( {create_int_key_pb(1), create_int_key_pb(2), create_int_key_pb(3)}); RowsetWriterContext writer_context; - create_rowset_writer_context(12345, partial_tablet_schema.get(), &writer_context); + create_rowset_writer_context(12345, partial_tablet_schema, &writer_context); writer_context.writer_type = kHorizontal; std::unique_ptr rowset_writer; @@ -914,7 +914,7 @@ TEST_F(RowsetTest, SegmentRewriterAutoIncrementTest) { std::vector> seg_infos; { std::vector column_indexes{0, 1, 2}; - auto schema = ChunkHelper::convert_schema(*partial_tablet_schema, column_indexes); + auto schema = ChunkHelper::convert_schema(partial_tablet_schema, column_indexes); auto chunk = ChunkHelper::new_chunk(schema, chunk_size); for (auto i = 0; i < num_rows / chunk_size + 1; ++i) { chunk->reset(); @@ -937,10 +937,10 @@ TEST_F(RowsetTest, SegmentRewriterAutoIncrementTest) { std::shared_ptr fs = FileSystem::CreateSharedFromString(rowset->rowset_path()).value(); std::string file_name = Rowset::segment_file_path(rowset->rowset_path(), rowset->rowset_id(), 0); - auto partial_segment = *Segment::open(fs, file_name, 0, partial_tablet_schema.get()); + auto partial_segment = *Segment::open(fs, file_name, 0, partial_tablet_schema); ASSERT_EQ(partial_segment->num_rows(), num_rows); - std::unique_ptr tablet_schema = TabletSchemaHelper::create_tablet_schema( + std::shared_ptr tablet_schema = TabletSchemaHelper::create_tablet_schema( {create_int_key_pb(1), create_int_key_pb(2), create_int_value_pb(3), create_int_value_pb(4)}); std::vector read_column_ids{2, 3}; std::vector> write_columns(read_column_ids.size()); @@ -955,16 +955,16 @@ TEST_F(RowsetTest, SegmentRewriterAutoIncrementTest) { } AutoIncrementPartialUpdateState auto_increment_partial_update_state; - auto_increment_partial_update_state.init(rowset.get(), partial_tablet_schema.get(), 2, 0); + auto_increment_partial_update_state.init(rowset.get(), partial_tablet_schema, 2, 0); auto_increment_partial_update_state.write_column.reset(write_columns[0].release()); write_columns.erase(write_columns.begin()); auto dst_file_name = Rowset::segment_temp_file_path(rowset->rowset_path(), rowset->rowset_id(), 0); std::vector column_ids{3}; - ASSERT_OK(SegmentRewriter::rewrite(file_name, dst_file_name, *tablet_schema, auto_increment_partial_update_state, + ASSERT_OK(SegmentRewriter::rewrite(file_name, dst_file_name, tablet_schema, auto_increment_partial_update_state, column_ids, &write_columns)); - auto segment = *Segment::open(fs, dst_file_name, 0, tablet_schema.get()); + auto segment = *Segment::open(fs, dst_file_name, 0, tablet_schema); ASSERT_EQ(segment->num_rows(), num_rows); } @@ -972,7 +972,7 @@ TEST_F(RowsetTest, SegmentDeleteWriteTest) { auto tablet = create_tablet(12345, 1111); int64_t num_rows = 1024; RowsetWriterContext writer_context; - create_rowset_writer_context(12345, &tablet->tablet_schema(), &writer_context); + create_rowset_writer_context(12345, tablet->tablet_schema(), &writer_context); std::unique_ptr rowset_writer; ASSERT_TRUE(RowsetFactory::create_rowset_writer(writer_context, &rowset_writer).ok()); diff --git a/be/test/storage/rowset/segment_iterator_test.cpp b/be/test/storage/rowset/segment_iterator_test.cpp index d108bc2eea2ee..ad965498df41f 100644 --- a/be/test/storage/rowset/segment_iterator_test.cpp +++ b/be/test/storage/rowset/segment_iterator_test.cpp @@ -68,7 +68,7 @@ TEST_F(SegmentIteratorTest, TestGlobalDictNotSuperSet) { ColumnPB c2 = create_with_default_value_pb("VARCHAR", ""); c2.set_length(128); - std::unique_ptr tablet_schema = TabletSchemaHelper::create_tablet_schema({c1, c2}); + std::shared_ptr tablet_schema = TabletSchemaHelper::create_tablet_schema({c1, c2}); SegmentWriterOptions opts; opts.num_rows_per_block = 10; @@ -76,7 +76,7 @@ TEST_F(SegmentIteratorTest, TestGlobalDictNotSuperSet) { std::string file_name = kSegmentDir + "/low_card_cols"; ASSIGN_OR_ABORT(auto wfile, _fs->new_writable_file(file_name)); - SegmentWriter writer(std::move(wfile), 0, tablet_schema.get(), opts); + SegmentWriter writer(std::move(wfile), 0, tablet_schema, opts); int32_t chunk_size = config::vector_chunk_size; size_t num_rows = 10000; @@ -87,7 +87,7 @@ TEST_F(SegmentIteratorTest, TestGlobalDictNotSuperSet) { // col0 std::vector column_indexes = {0}; ASSERT_OK(writer.init(column_indexes, true)); - auto schema = ChunkHelper::convert_schema(*tablet_schema, column_indexes); + auto schema = ChunkHelper::convert_schema(tablet_schema, column_indexes); auto chunk = ChunkHelper::new_chunk(schema, chunk_size); for (auto i = 0; i < num_rows % chunk_size; ++i) { chunk->reset(); @@ -103,7 +103,7 @@ TEST_F(SegmentIteratorTest, TestGlobalDictNotSuperSet) { // col1 std::vector column_indexes{1}; ASSERT_OK(writer.init(column_indexes, false)); - auto schema = ChunkHelper::convert_schema(*tablet_schema, column_indexes); + auto schema = ChunkHelper::convert_schema(tablet_schema, column_indexes); auto chunk = ChunkHelper::new_chunk(schema, chunk_size); for (auto i = 0; i < num_rows % chunk_size; ++i) { chunk->reset(); @@ -117,7 +117,7 @@ TEST_F(SegmentIteratorTest, TestGlobalDictNotSuperSet) { } ASSERT_OK(writer.finalize_footer(&file_size)); - auto segment = *Segment::open(_fs, file_name, 0, tablet_schema.get()); + auto segment = *Segment::open(_fs, file_name, 0, tablet_schema); ASSERT_EQ(segment->num_rows(), num_rows); SegmentReadOptions seg_options; @@ -192,7 +192,7 @@ TEST_F(SegmentIteratorTest, TestGlobalDictNoLocalDict) { ColumnPB c2 = create_with_default_value_pb("VARCHAR", ""); c2.set_length(overflow_sz + 10); - std::unique_ptr tablet_schema = TabletSchemaHelper::create_tablet_schema({c1, c2}); + std::shared_ptr tablet_schema = TabletSchemaHelper::create_tablet_schema({c1, c2}); SegmentWriterOptions opts; opts.num_rows_per_block = 1024; @@ -200,7 +200,7 @@ TEST_F(SegmentIteratorTest, TestGlobalDictNoLocalDict) { std::string file_name = kSegmentDir + "/no_dict"; ASSIGN_OR_ABORT(auto wfile, _fs->new_writable_file(file_name)); - SegmentWriter writer(std::move(wfile), 0, tablet_schema.get(), opts); + SegmentWriter writer(std::move(wfile), 0, tablet_schema, opts); int32_t chunk_size = config::vector_chunk_size; size_t num_rows = slice_num; @@ -211,7 +211,7 @@ TEST_F(SegmentIteratorTest, TestGlobalDictNoLocalDict) { // col0 std::vector column_indexes = {0}; ASSERT_OK(writer.init(column_indexes, true)); - auto schema = ChunkHelper::convert_schema(*tablet_schema, column_indexes); + auto schema = ChunkHelper::convert_schema(tablet_schema, column_indexes); auto chunk = ChunkHelper::new_chunk(schema, chunk_size); for (auto i = 0; i < num_rows % chunk_size; ++i) { chunk->reset(); @@ -227,7 +227,7 @@ TEST_F(SegmentIteratorTest, TestGlobalDictNoLocalDict) { // col1 std::vector column_indexes{1}; ASSERT_OK(writer.init(column_indexes, false)); - auto schema = ChunkHelper::convert_schema(*tablet_schema, column_indexes); + auto schema = ChunkHelper::convert_schema(tablet_schema, column_indexes); auto chunk = ChunkHelper::new_chunk(schema, chunk_size); for (auto i = 0; i < num_rows % chunk_size; ++i) { chunk->reset(); @@ -241,7 +241,7 @@ TEST_F(SegmentIteratorTest, TestGlobalDictNoLocalDict) { } ASSERT_OK(writer.finalize_footer(&file_size)); - auto segment = *Segment::open(_fs, file_name, 0, tablet_schema.get()); + auto segment = *Segment::open(_fs, file_name, 0, tablet_schema); ASSERT_EQ(segment->num_rows(), num_rows); SegmentReadOptions seg_options; diff --git a/be/test/storage/rowset/segment_rewriter_test.cpp b/be/test/storage/rowset/segment_rewriter_test.cpp index b29a88015dddb..71acea5ab07ac 100644 --- a/be/test/storage/rowset/segment_rewriter_test.cpp +++ b/be/test/storage/rowset/segment_rewriter_test.cpp @@ -66,7 +66,7 @@ class SegmentRewriterTest : public ::testing::Test { }; TEST_F(SegmentRewriterTest, rewrite_test) { - std::unique_ptr partial_tablet_schema = TabletSchemaHelper::create_tablet_schema( + std::shared_ptr partial_tablet_schema = TabletSchemaHelper::create_tablet_schema( {create_int_key_pb(1), create_int_key_pb(2), create_int_value_pb(4)}); SegmentWriterOptions opts; @@ -75,12 +75,12 @@ TEST_F(SegmentRewriterTest, rewrite_test) { std::string file_name = kSegmentDir + "/partial_rowset"; ASSIGN_OR_ABORT(auto wfile, _fs->new_writable_file(file_name)); - SegmentWriter writer(std::move(wfile), 0, partial_tablet_schema.get(), opts); + SegmentWriter writer(std::move(wfile), 0, partial_tablet_schema, opts); ASSERT_OK(writer.init()); int32_t chunk_size = config::vector_chunk_size; size_t num_rows = 10000; - auto partial_schema = ChunkHelper::convert_schema(*partial_tablet_schema); + auto partial_schema = ChunkHelper::convert_schema(partial_tablet_schema); auto partial_chunk = ChunkHelper::new_chunk(partial_schema, chunk_size); for (auto i = 0; i < num_rows % chunk_size; ++i) { @@ -103,10 +103,10 @@ TEST_F(SegmentRewriterTest, rewrite_test) { partial_rowset_footer.set_position(footer_position); partial_rowset_footer.set_size(file_size - footer_position); - auto partial_segment = *Segment::open(_fs, file_name, 0, partial_tablet_schema.get()); + auto partial_segment = *Segment::open(_fs, file_name, 0, partial_tablet_schema); ASSERT_EQ(partial_segment->num_rows(), num_rows); - std::unique_ptr tablet_schema = TabletSchemaHelper::create_tablet_schema( + std::shared_ptr tablet_schema = TabletSchemaHelper::create_tablet_schema( {create_int_key_pb(1), create_int_key_pb(2), create_int_value_pb(3), create_int_value_pb(4), create_int_value_pb(5)}); std::string dst_file_name = kSegmentDir + "/rewrite_rowset"; @@ -122,17 +122,17 @@ TEST_F(SegmentRewriterTest, rewrite_test) { } } - ASSERT_OK(SegmentRewriter::rewrite(file_name, dst_file_name, *tablet_schema, read_column_ids, write_columns, + ASSERT_OK(SegmentRewriter::rewrite(file_name, dst_file_name, tablet_schema, read_column_ids, write_columns, partial_segment->id(), partial_rowset_footer)); - auto segment = *Segment::open(_fs, dst_file_name, 0, tablet_schema.get()); + auto segment = *Segment::open(_fs, dst_file_name, 0, tablet_schema); ASSERT_EQ(segment->num_rows(), num_rows); SegmentReadOptions seg_options; seg_options.fs = _fs; OlapReaderStatistics stats; seg_options.stats = &stats; - auto schema = ChunkHelper::convert_schema(*tablet_schema); + auto schema = ChunkHelper::convert_schema(tablet_schema); auto res = segment->new_iterator(schema, seg_options); ASSERT_FALSE(res.status().is_end_of_file() || !res.ok() || res.value() == nullptr); auto seg_iterator = res.value(); @@ -174,9 +174,9 @@ TEST_F(SegmentRewriterTest, rewrite_test) { new_write_columns[i]->append_datum(Datum(static_cast(j + read_column_ids[i]))); } } - ASSERT_OK(SegmentRewriter::rewrite(file_name, *tablet_schema, read_column_ids, new_write_columns, + ASSERT_OK(SegmentRewriter::rewrite(file_name, tablet_schema, read_column_ids, new_write_columns, partial_segment->id(), partial_rowset_footer)); - auto rewrite_segment = *Segment::open(_fs, file_name, 0, tablet_schema.get()); + auto rewrite_segment = *Segment::open(_fs, file_name, 0, tablet_schema); ASSERT_EQ(rewrite_segment->num_rows(), num_rows); res = rewrite_segment->new_iterator(schema, seg_options); diff --git a/be/test/storage/rowset/segment_test.cpp b/be/test/storage/rowset/segment_test.cpp index bfe4cee3ddaac..8d8cdf65fb193 100644 --- a/be/test/storage/rowset/segment_test.cpp +++ b/be/test/storage/rowset/segment_test.cpp @@ -83,22 +83,22 @@ class SegmentReaderWriterTest : public ::testing::Test { void TearDown() override { StoragePageCache::release_global_cache(); } - void build_segment(const SegmentWriterOptions& opts, const TabletSchema& build_schema, - const TabletSchema& query_schema, size_t nrows, const ValueGenerator& generator, + void build_segment(const SegmentWriterOptions& opts, const TabletSchemaCSPtr& build_schema, + const TabletSchemaCSPtr& query_schema, size_t nrows, const ValueGenerator& generator, shared_ptr* res) { static int seg_id = 0; // must use unique filename for each segment, otherwise page cache kicks in and produces // the wrong answer (it use (filename,offset) as cache key) std::string filename = strings::Substitute("$0/seg_$1.dat", kSegmentDir, seg_id++); ASSIGN_OR_ABORT(auto wfile, _fs->new_writable_file(filename)); - SegmentWriter writer(std::move(wfile), 0, &build_schema, opts); + SegmentWriter writer(std::move(wfile), 0, build_schema, opts); ASSERT_OK(writer.init()); auto schema = ChunkHelper::convert_schema(build_schema); auto chunk = ChunkHelper::new_chunk(schema, nrows); for (size_t rid = 0; rid < nrows; ++rid) { auto& cols = chunk->columns(); - for (int cid = 0; cid < build_schema.num_columns(); ++cid) { + for (int cid = 0; cid < build_schema->num_columns(); ++cid) { int row_block_id = rid / opts.num_rows_per_block; cols[cid]->append_datum(generator(rid, cid, row_block_id)); } @@ -108,7 +108,7 @@ class SegmentReaderWriterTest : public ::testing::Test { uint64_t file_size, index_size, footer_position; ASSERT_OK(writer.finalize(&file_size, &index_size, &footer_position)); - *res = *Segment::open(_fs, filename, 0, &query_schema); + *res = *Segment::open(_fs, filename, 0, query_schema); ASSERT_EQ(nrows, (*res)->num_rows()); } @@ -121,7 +121,7 @@ class SegmentReaderWriterTest : public ::testing::Test { TEST_F(SegmentReaderWriterTest, estimate_segment_size) { size_t num_rows_per_block = 10; - auto tablet_schema = TabletSchemaHelper::create_tablet_schema( + std::shared_ptr tablet_schema = TabletSchemaHelper::create_tablet_schema( {create_int_key_pb(1), create_int_key_pb(2), create_int_key_pb(3), create_int_value_pb(4)}, 2); tablet_schema->_num_rows_per_row_block = 2; @@ -134,14 +134,14 @@ TEST_F(SegmentReaderWriterTest, estimate_segment_size) { std::string fname = dname + "/int_case"; ASSIGN_OR_ABORT(auto wfile, _fs->new_writable_file(fname)); - SegmentWriter writer(std::move(wfile), 0, tablet_schema.get(), opts); + SegmentWriter writer(std::move(wfile), 0, tablet_schema, opts); ASSERT_OK(writer.init()); // 0, 1, 2, 3 // 10, 11, 12, 13 // 20, 21, 22, 23 size_t nrows = 1048576; - auto schema = ChunkHelper::convert_schema(*tablet_schema); + auto schema = ChunkHelper::convert_schema(tablet_schema); auto chunk = ChunkHelper::new_chunk(schema, nrows); for (size_t rid = 0; rid < nrows; ++rid) { auto& cols = chunk->columns(); @@ -166,25 +166,25 @@ TEST_F(SegmentReaderWriterTest, estimate_segment_size) { } TEST_F(SegmentReaderWriterTest, TestBloomFilterIndexUniqueModel) { - std::unique_ptr schema = + std::shared_ptr schema = TabletSchemaHelper::create_tablet_schema({create_int_key_pb(1), create_int_key_pb(2), create_int_key_pb(3), create_int_value_pb(4, "REPLACE", true, "", true)}); // for not base segment SegmentWriterOptions opts1; shared_ptr seg1; - build_segment(opts1, *schema, *schema, 100, DefaultIntGenerator, &seg1); + build_segment(opts1, schema, schema, 100, DefaultIntGenerator, &seg1); ASSERT_TRUE(seg1->column(3)->has_bloom_filter_index()); // for base segment SegmentWriterOptions opts2; shared_ptr seg2; - build_segment(opts2, *schema, *schema, 100, DefaultIntGenerator, &seg2); + build_segment(opts2, schema, schema, 100, DefaultIntGenerator, &seg2); ASSERT_TRUE(seg2->column(3)->has_bloom_filter_index()); } TEST_F(SegmentReaderWriterTest, TestHorizontalWrite) { - std::unique_ptr tablet_schema = TabletSchemaHelper::create_tablet_schema( + std::shared_ptr tablet_schema = TabletSchemaHelper::create_tablet_schema( {create_int_key_pb(1), create_int_key_pb(2), create_int_value_pb(3), create_int_value_pb(4)}); SegmentWriterOptions opts; @@ -193,12 +193,12 @@ TEST_F(SegmentReaderWriterTest, TestHorizontalWrite) { std::string file_name = kSegmentDir + "/horizontal_write_case"; ASSIGN_OR_ABORT(auto wfile, _fs->new_writable_file(file_name)); - SegmentWriter writer(std::move(wfile), 0, tablet_schema.get(), opts); + SegmentWriter writer(std::move(wfile), 0, tablet_schema, opts); ASSERT_OK(writer.init()); int32_t chunk_size = config::vector_chunk_size; size_t num_rows = 10000; - auto schema = ChunkHelper::convert_schema(*tablet_schema); + auto schema = ChunkHelper::convert_schema(tablet_schema); auto chunk = ChunkHelper::new_chunk(schema, chunk_size); for (auto i = 0; i < num_rows % chunk_size; ++i) { chunk->reset(); @@ -217,7 +217,7 @@ TEST_F(SegmentReaderWriterTest, TestHorizontalWrite) { uint64_t footer_position; ASSERT_OK(writer.finalize(&file_size, &index_size, &footer_position)); - auto segment = *Segment::open(_fs, file_name, 0, tablet_schema.get()); + auto segment = *Segment::open(_fs, file_name, 0, tablet_schema); ASSERT_EQ(segment->num_rows(), num_rows); SegmentReadOptions seg_options; @@ -249,7 +249,7 @@ TEST_F(SegmentReaderWriterTest, TestHorizontalWrite) { // NOLINTNEXTLINE TEST_F(SegmentReaderWriterTest, TestVerticalWrite) { - std::unique_ptr tablet_schema = TabletSchemaHelper::create_tablet_schema( + std::shared_ptr tablet_schema = TabletSchemaHelper::create_tablet_schema( {create_int_key_pb(1), create_int_key_pb(2), create_int_value_pb(3), create_int_value_pb(4)}); SegmentWriterOptions opts; @@ -258,7 +258,7 @@ TEST_F(SegmentReaderWriterTest, TestVerticalWrite) { std::string file_name = kSegmentDir + "/vertical_write_case"; ASSIGN_OR_ABORT(auto wfile, _fs->new_writable_file(file_name)); - SegmentWriter writer(std::move(wfile), 0, tablet_schema.get(), opts); + SegmentWriter writer(std::move(wfile), 0, tablet_schema, opts); int32_t chunk_size = config::vector_chunk_size; size_t num_rows = 10000; @@ -269,7 +269,7 @@ TEST_F(SegmentReaderWriterTest, TestVerticalWrite) { // col1 col2 std::vector column_indexes{0, 1}; ASSERT_OK(writer.init(column_indexes, true)); - auto schema = ChunkHelper::convert_schema(*tablet_schema, column_indexes); + auto schema = ChunkHelper::convert_schema(tablet_schema, column_indexes); auto chunk = ChunkHelper::new_chunk(schema, chunk_size); for (auto i = 0; i < num_rows % chunk_size; ++i) { chunk->reset(); @@ -287,7 +287,7 @@ TEST_F(SegmentReaderWriterTest, TestVerticalWrite) { // col3 std::vector column_indexes{2}; ASSERT_OK(writer.init(column_indexes, false)); - auto schema = ChunkHelper::convert_schema(*tablet_schema, column_indexes); + auto schema = ChunkHelper::convert_schema(tablet_schema, column_indexes); auto chunk = ChunkHelper::new_chunk(schema, chunk_size); for (auto i = 0; i < num_rows % chunk_size; ++i) { chunk->reset(); @@ -304,7 +304,7 @@ TEST_F(SegmentReaderWriterTest, TestVerticalWrite) { // col4 std::vector column_indexes{3}; ASSERT_OK(writer.init(column_indexes, false)); - auto schema = ChunkHelper::convert_schema(*tablet_schema, column_indexes); + auto schema = ChunkHelper::convert_schema(tablet_schema, column_indexes); auto chunk = ChunkHelper::new_chunk(schema, chunk_size); for (auto i = 0; i < num_rows % chunk_size; ++i) { chunk->reset(); @@ -319,14 +319,14 @@ TEST_F(SegmentReaderWriterTest, TestVerticalWrite) { ASSERT_OK(writer.finalize_footer(&file_size)); - auto segment = *Segment::open(_fs, file_name, 0, tablet_schema.get()); + auto segment = *Segment::open(_fs, file_name, 0, tablet_schema); ASSERT_EQ(segment->num_rows(), num_rows); SegmentReadOptions seg_options; seg_options.fs = _fs; OlapReaderStatistics stats; seg_options.stats = &stats; - auto schema = ChunkHelper::convert_schema(*tablet_schema); + auto schema = ChunkHelper::convert_schema(tablet_schema); auto res = segment->new_iterator(schema, seg_options); ASSERT_FALSE(res.status().is_end_of_file() || !res.ok() || res.value() == nullptr); auto seg_iterator = res.value(); @@ -367,7 +367,7 @@ TEST_F(SegmentReaderWriterTest, TestReadMultipleTypesColumn) { ColumnPB c3 = create_with_default_value_pb("VARCHAR", ""); c3.set_length(65535); - std::unique_ptr tablet_schema = TabletSchemaHelper::create_tablet_schema({c1, c2, c3}); + std::shared_ptr tablet_schema = TabletSchemaHelper::create_tablet_schema({c1, c2, c3}); SegmentWriterOptions opts; opts.num_rows_per_block = 10; @@ -375,7 +375,7 @@ TEST_F(SegmentReaderWriterTest, TestReadMultipleTypesColumn) { std::string file_name = kSegmentDir + "/read_multiple_types_column"; ASSIGN_OR_ABORT(auto wfile, _fs->new_writable_file(file_name)); - SegmentWriter writer(std::move(wfile), 0, tablet_schema.get(), opts); + SegmentWriter writer(std::move(wfile), 0, tablet_schema, opts); int32_t chunk_size = config::vector_chunk_size; size_t num_rows = 10000; @@ -386,7 +386,7 @@ TEST_F(SegmentReaderWriterTest, TestReadMultipleTypesColumn) { // col1 col2 std::vector column_indexes{0, 1}; ASSERT_OK(writer.init(column_indexes, true)); - auto schema = ChunkHelper::convert_schema(*tablet_schema, column_indexes); + auto schema = ChunkHelper::convert_schema(tablet_schema, column_indexes); auto chunk = ChunkHelper::new_chunk(schema, chunk_size); for (auto i = 0; i < num_rows % chunk_size; ++i) { chunk->reset(); @@ -404,7 +404,7 @@ TEST_F(SegmentReaderWriterTest, TestReadMultipleTypesColumn) { // col3 std::vector column_indexes{2}; ASSERT_OK(writer.init(column_indexes, false)); - auto schema = ChunkHelper::convert_schema(*tablet_schema, column_indexes); + auto schema = ChunkHelper::convert_schema(tablet_schema, column_indexes); auto chunk = ChunkHelper::new_chunk(schema, chunk_size); for (auto i = 0; i < num_rows % chunk_size; ++i) { chunk->reset(); @@ -418,14 +418,14 @@ TEST_F(SegmentReaderWriterTest, TestReadMultipleTypesColumn) { } ASSERT_OK(writer.finalize_footer(&file_size)); - auto segment = *Segment::open(_fs, file_name, 0, tablet_schema.get()); + auto segment = *Segment::open(_fs, file_name, 0, tablet_schema); ASSERT_EQ(segment->num_rows(), num_rows); SegmentReadOptions seg_options; seg_options.fs = _fs; OlapReaderStatistics stats; seg_options.stats = &stats; - auto schema = ChunkHelper::convert_schema(*tablet_schema); + auto schema = ChunkHelper::convert_schema(tablet_schema); auto res = segment->new_iterator(schema, seg_options); ASSERT_FALSE(res.status().is_end_of_file() || !res.ok() || res.value() == nullptr); auto seg_iterator = res.value(); diff --git a/be/test/storage/rowset/struct_column_rw_test.cpp b/be/test/storage/rowset/struct_column_rw_test.cpp index 35b5f21bee581..9b3b6912526ae 100644 --- a/be/test/storage/rowset/struct_column_rw_test.cpp +++ b/be/test/storage/rowset/struct_column_rw_test.cpp @@ -50,7 +50,7 @@ class StructColumnRWTest : public testing::Test { void TearDown() override {} std::shared_ptr create_dummy_segment(const std::shared_ptr& fs, const std::string& fname) { - return std::make_shared(Segment::private_type(0), fs, fname, 1, _dummy_segment_schema.get()); + return std::make_shared(Segment::private_type(0), fs, fname, 1, _dummy_segment_schema); } void test_int_struct() { diff --git a/be/test/storage/rowset_column_partial_update_test.cpp b/be/test/storage/rowset_column_partial_update_test.cpp index 665d17563fad6..21363830d0775 100644 --- a/be/test/storage/rowset_column_partial_update_test.cpp +++ b/be/test/storage/rowset_column_partial_update_test.cpp @@ -66,7 +66,7 @@ class RowsetColumnPartialUpdateTest : public ::testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; @@ -156,14 +156,14 @@ class RowsetColumnPartialUpdateTest : public ::testing::Test { writer_context.partial_update_tablet_schema = partial_schema; writer_context.referenced_column_ids = column_indexes; - writer_context.tablet_schema = partial_schema.get(); + writer_context.tablet_schema = partial_schema; writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; writer_context.partial_update_mode = mode; std::unique_ptr writer; EXPECT_TRUE(RowsetFactory::create_rowset_writer(writer_context, &writer).ok()); - auto schema = ChunkHelper::convert_schema(*partial_schema.get()); + auto schema = ChunkHelper::convert_schema(partial_schema); auto chunk = ChunkHelper::new_chunk(schema, keys.size()); EXPECT_TRUE(2 == chunk->num_columns()); @@ -183,7 +183,7 @@ class RowsetColumnPartialUpdateTest : public ::testing::Test { CHECK_OK(writer->flush_chunk(*chunk)); } RowsetSharedPtr partial_rowset = *writer->build(); - partial_rowset->set_schema(&tablet->tablet_schema()); + partial_rowset->set_schema(tablet->tablet_schema()); return partial_rowset; } diff --git a/be/test/storage/rowset_column_update_state_test.cpp b/be/test/storage/rowset_column_update_state_test.cpp index 9aa7ca71c1ed7..663624dae4821 100644 --- a/be/test/storage/rowset_column_update_state_test.cpp +++ b/be/test/storage/rowset_column_update_state_test.cpp @@ -64,7 +64,7 @@ class RowsetColumnUpdateStateTest : public ::testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; @@ -135,14 +135,14 @@ class RowsetColumnUpdateStateTest : public ::testing::Test { writer_context.partial_update_tablet_schema = partial_schema; writer_context.referenced_column_ids = column_indexes; - writer_context.tablet_schema = partial_schema.get(); + writer_context.tablet_schema = partial_schema; writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; writer_context.partial_update_mode = PartialUpdateMode::COLUMN_UPDATE_MODE; std::unique_ptr writer; EXPECT_TRUE(RowsetFactory::create_rowset_writer(writer_context, &writer).ok()); - auto schema = ChunkHelper::convert_schema(*partial_schema.get()); + auto schema = ChunkHelper::convert_schema(partial_schema); auto chunk = ChunkHelper::new_chunk(schema, keys.size()); EXPECT_TRUE(2 == chunk->num_columns()); diff --git a/be/test/storage/rowset_merger_test.cpp b/be/test/storage/rowset_merger_test.cpp index e17d5ba47cc0d..7af028298e9d6 100644 --- a/be/test/storage/rowset_merger_test.cpp +++ b/be/test/storage/rowset_merger_test.cpp @@ -104,7 +104,7 @@ class RowsetMergerTest : public testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; diff --git a/be/test/storage/rowset_update_state_test.cpp b/be/test/storage/rowset_update_state_test.cpp index e533f170655bc..29466615a26fa 100644 --- a/be/test/storage/rowset_update_state_test.cpp +++ b/be/test/storage/rowset_update_state_test.cpp @@ -64,7 +64,7 @@ class RowsetUpdateStateTest : public ::testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; @@ -135,13 +135,13 @@ class RowsetUpdateStateTest : public ::testing::Test { writer_context.partial_update_tablet_schema = partial_schema; writer_context.referenced_column_ids = column_indexes; - writer_context.tablet_schema = partial_schema.get(); + writer_context.tablet_schema = partial_schema; writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; std::unique_ptr writer; EXPECT_TRUE(RowsetFactory::create_rowset_writer(writer_context, &writer).ok()); - auto schema = ChunkHelper::convert_schema(*partial_schema.get()); + auto schema = ChunkHelper::convert_schema(partial_schema); auto chunk = ChunkHelper::new_chunk(schema, keys.size()); EXPECT_TRUE(2 == chunk->num_columns()); @@ -291,7 +291,7 @@ TEST_F(RowsetUpdateStateTest, check_conflict) { writer_context.partition_id = 0; writer_context.rowset_path_prefix = _tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &_tablet->tablet_schema(); + writer_context.tablet_schema = _tablet->tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; diff --git a/be/test/storage/schema_change_test.cpp b/be/test/storage/schema_change_test.cpp index d12f235a71ed4..d1642ee3034d0 100644 --- a/be/test/storage/schema_change_test.cpp +++ b/be/test/storage/schema_change_test.cpp @@ -93,7 +93,7 @@ class SchemaChangeTest : public testing::Test { writer_context.tablet_id = tablet->tablet_id(); writer_context.tablet_schema_hash = tablet->schema_hash(); writer_context.rowset_path_prefix = tablet->schema_hash_path(); - writer_context.tablet_schema = &(tablet->tablet_schema()); + writer_context.tablet_schema = tablet->tablet_schema(); writer_context.rowset_state = VISIBLE; if (version == nullptr) { writer_context.version = Version(3, 3); @@ -343,8 +343,8 @@ TEST_F(SchemaChangeTest, convert_int_to_bitmap) { auto src_tablet_schema = SetTabletSchema("IntColumn", "INT", "REPLACE", 4, false, false); auto dst_tablet_schema = SetTabletSchema("BitmapColumn", "OBJECT", "BITMAP_UNION", 8, false, false); - ChunkPtr src_chunk = ChunkHelper::new_chunk(ChunkHelper::convert_schema(*src_tablet_schema), 4096); - ChunkPtr dst_chunk = ChunkHelper::new_chunk(ChunkHelper::convert_schema(*dst_tablet_schema), 4096); + ChunkPtr src_chunk = ChunkHelper::new_chunk(ChunkHelper::convert_schema(src_tablet_schema), 4096); + ChunkPtr dst_chunk = ChunkHelper::new_chunk(ChunkHelper::convert_schema(dst_tablet_schema), 4096); ColumnPtr& src_col = src_chunk->get_column_by_index(0); ColumnPtr& dst_col = dst_chunk->get_column_by_index(0); Field f = ChunkHelper::convert_field(0, src_tablet_schema->column(0)); @@ -367,8 +367,8 @@ TEST_F(SchemaChangeTest, convert_varchar_to_hll) { auto src_tablet_schema = SetTabletSchema("IntColumn", "VARCHAR", "REPLACE", 255, false, false); auto dst_tablet_schema = SetTabletSchema("HLLColumn", "HLL", "HLL_UNION", 8, false, false); - ChunkPtr src_chunk = ChunkHelper::new_chunk(ChunkHelper::convert_schema(*src_tablet_schema), 4096); - ChunkPtr dst_chunk = ChunkHelper::new_chunk(ChunkHelper::convert_schema(*dst_tablet_schema), 4096); + ChunkPtr src_chunk = ChunkHelper::new_chunk(ChunkHelper::convert_schema(src_tablet_schema), 4096); + ChunkPtr dst_chunk = ChunkHelper::new_chunk(ChunkHelper::convert_schema(dst_tablet_schema), 4096); ColumnPtr& src_col = src_chunk->get_column_by_index(0); ColumnPtr& dst_col = dst_chunk->get_column_by_index(0); Field f = ChunkHelper::convert_field(0, src_tablet_schema->column(0)); @@ -393,8 +393,8 @@ TEST_F(SchemaChangeTest, convert_int_to_count) { auto src_tablet_schema = SetTabletSchema("IntColumn", "INT", "REPLACE", 4, false, false); auto dst_tablet_schema = SetTabletSchema("CountColumn", "BIGINT", "SUM", 8, false, false); - ChunkPtr src_chunk = ChunkHelper::new_chunk(ChunkHelper::convert_schema(*src_tablet_schema), 4096); - ChunkPtr dst_chunk = ChunkHelper::new_chunk(ChunkHelper::convert_schema(*dst_tablet_schema), 4096); + ChunkPtr src_chunk = ChunkHelper::new_chunk(ChunkHelper::convert_schema(src_tablet_schema), 4096); + ChunkPtr dst_chunk = ChunkHelper::new_chunk(ChunkHelper::convert_schema(dst_tablet_schema), 4096); ColumnPtr& src_col = src_chunk->get_column_by_index(0); ColumnPtr& dst_col = dst_chunk->get_column_by_index(0); Field f = ChunkHelper::convert_field(0, src_tablet_schema->column(0)); @@ -454,7 +454,7 @@ TEST_F(SchemaChangeTest, schema_change_with_directing_v2) { writer_context.tablet_id = new_tablet->tablet_id(); writer_context.tablet_schema_hash = new_tablet->schema_hash(); writer_context.rowset_path_prefix = new_tablet->schema_hash_path(); - writer_context.tablet_schema = &(new_tablet->tablet_schema()); + writer_context.tablet_schema = new_tablet->tablet_schema(); writer_context.rowset_state = VISIBLE; writer_context.version = Version(3, 3); std::unique_ptr rowset_writer; @@ -516,7 +516,7 @@ TEST_F(SchemaChangeTest, schema_change_with_sorting_v2) { writer_context.tablet_id = new_tablet->tablet_id(); writer_context.tablet_schema_hash = new_tablet->schema_hash(); writer_context.rowset_path_prefix = new_tablet->schema_hash_path(); - writer_context.tablet_schema = &(new_tablet->tablet_schema()); + writer_context.tablet_schema = new_tablet->tablet_schema(); writer_context.rowset_state = VISIBLE; writer_context.version = Version(3, 3); std::unique_ptr rowset_writer; @@ -575,7 +575,7 @@ TEST_F(SchemaChangeTest, schema_change_with_agg_key_reorder) { writer_context.tablet_id = new_tablet->tablet_id(); writer_context.tablet_schema_hash = new_tablet->schema_hash(); writer_context.rowset_path_prefix = new_tablet->schema_hash_path(); - writer_context.tablet_schema = &(new_tablet->tablet_schema()); + writer_context.tablet_schema = new_tablet->tablet_schema(); writer_context.rowset_state = VISIBLE; writer_context.version = Version(3, 3); std::unique_ptr rowset_writer; @@ -698,7 +698,7 @@ TEST_F(SchemaChangeTest, schema_change_with_materialized_column_old_style) { writer_context.tablet_id = new_tablet->tablet_id(); writer_context.tablet_schema_hash = new_tablet->schema_hash(); writer_context.rowset_path_prefix = new_tablet->schema_hash_path(); - writer_context.tablet_schema = &(new_tablet->tablet_schema()); + writer_context.tablet_schema = new_tablet->tablet_schema(); writer_context.rowset_state = VISIBLE; writer_context.version = Version(3, 3); std::unique_ptr rowset_writer; diff --git a/be/test/storage/size_tiered_compaction_policy_test.cpp b/be/test/storage/size_tiered_compaction_policy_test.cpp index 44257cd355435..50575e5209227 100644 --- a/be/test/storage/size_tiered_compaction_policy_test.cpp +++ b/be/test/storage/size_tiered_compaction_policy_test.cpp @@ -54,7 +54,7 @@ class SizeTieredCompactionPolicyTest : public testing::Test { void rowset_writer_add_rows(std::unique_ptr& writer, int64_t level) { std::srand(std::time(nullptr)); std::vector test_data; - auto schema = ChunkHelper::convert_schema(*_tablet_schema); + auto schema = ChunkHelper::convert_schema(_tablet_schema); auto chunk = ChunkHelper::new_chunk(schema, 1024); for (size_t i = 0; i < 24576 * pow(config::size_tiered_level_multiple + 1, level - 2); ++i) { test_data.push_back("well" + std::to_string(std::rand())); @@ -160,7 +160,7 @@ class SizeTieredCompactionPolicyTest : public testing::Test { rowset_writer_context->partition_id = 10; rowset_writer_context->rowset_path_prefix = config::storage_root_path + "/data/0/12345/1111"; rowset_writer_context->rowset_state = VISIBLE; - rowset_writer_context->tablet_schema = _tablet_schema.get(); + rowset_writer_context->tablet_schema = _tablet_schema; rowset_writer_context->version.first = version; rowset_writer_context->version.second = version; } @@ -326,7 +326,7 @@ class SizeTieredCompactionPolicyTest : public testing::Test { protected: StorageEngine* _engine = nullptr; - std::unique_ptr _tablet_schema; + std::shared_ptr _tablet_schema; std::string _schema_hash_path; std::unique_ptr _metadata_mem_tracker; std::unique_ptr _compaction_mem_tracker; diff --git a/be/test/storage/table_reader_remote_test.cpp b/be/test/storage/table_reader_remote_test.cpp index f589e62e97ad9..63f248011a449 100644 --- a/be/test/storage/table_reader_remote_test.cpp +++ b/be/test/storage/table_reader_remote_test.cpp @@ -62,7 +62,7 @@ class TableReaderRemoteTest : public testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; diff --git a/be/test/storage/table_reader_test.cpp b/be/test/storage/table_reader_test.cpp index c72d166d35098..d95a567b4342c 100644 --- a/be/test/storage/table_reader_test.cpp +++ b/be/test/storage/table_reader_test.cpp @@ -64,7 +64,7 @@ class TableReaderTest : public testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; diff --git a/be/test/storage/tablet_binlog_test.cpp b/be/test/storage/tablet_binlog_test.cpp index 1e4ca7a3491df..3fa0e4b71cbb8 100644 --- a/be/test/storage/tablet_binlog_test.cpp +++ b/be/test/storage/tablet_binlog_test.cpp @@ -70,7 +70,7 @@ class TabletBinlogTest : public BinlogTestBase { writer_context.partition_id = 5; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; diff --git a/be/test/storage/tablet_mgr_test.cpp b/be/test/storage/tablet_mgr_test.cpp index ce0bfaad7f7e7..174a1dd56aaae 100644 --- a/be/test/storage/tablet_mgr_test.cpp +++ b/be/test/storage/tablet_mgr_test.cpp @@ -347,7 +347,7 @@ TEST_F(TabletMgrTest, GetNextBatchTabletsTest) { } static void create_rowset_writer_context(RowsetWriterContext* rowset_writer_context, - const std::string& schema_hash_path, const TabletSchema* tablet_schema, + const std::string& schema_hash_path, const TabletSchemaCSPtr tablet_schema, int64_t start_ver, int64_t end_ver, int64_t rid) { RowsetId rowset_id; rowset_id.init(rid); @@ -362,7 +362,7 @@ static void create_rowset_writer_context(RowsetWriterContext* rowset_writer_cont rowset_writer_context->version.second = end_ver; } -static void rowset_writer_add_rows(std::unique_ptr& writer, const TabletSchema& tablet_schema) { +static void rowset_writer_add_rows(std::unique_ptr& writer, const TabletSchemaCSPtr& tablet_schema) { std::vector test_data; auto schema = ChunkHelper::convert_schema(tablet_schema); auto chunk = ChunkHelper::new_chunk(schema, 1024); @@ -417,7 +417,7 @@ TEST_F(TabletMgrTest, RsVersionMapTest) { TabletManager* tablet_manager = starrocks::StorageEngine::instance()->tablet_manager(); TabletSharedPtr tablet = tablet_manager->get_tablet(12347); ASSERT_TRUE(tablet != nullptr); - const TabletSchema& tablet_schema = tablet->tablet_schema(); + const auto& tablet_schema = tablet->tablet_schema(); // create rowset <2, 2>, <3, 3>, <3, 4>, <4, 4>, <4, 5>, <5, 5>, <5, 6> std::vector ver_list; @@ -442,7 +442,7 @@ TEST_F(TabletMgrTest, RsVersionMapTest) { int64_t rid = 10000; for (auto&& ver : ver_list) { RowsetWriterContext rowset_writer_context; - create_rowset_writer_context(&rowset_writer_context, tablet->schema_hash_path(), &tablet_schema, ver.first, + create_rowset_writer_context(&rowset_writer_context, tablet->schema_hash_path(), tablet_schema, ver.first, ver.second, rid++); std::unique_ptr rowset_writer; ASSERT_TRUE(RowsetFactory::create_rowset_writer(rowset_writer_context, &rowset_writer).ok()); @@ -489,7 +489,7 @@ TEST_F(TabletMgrTest, RsVersionMapTest) { to_remove.clear(); for (int i = 0; i < 3; i++) { RowsetWriterContext rowset_writer_context; - create_rowset_writer_context(&rowset_writer_context, tablet->schema_hash_path(), &tablet_schema, + create_rowset_writer_context(&rowset_writer_context, tablet->schema_hash_path(), tablet_schema, ver_list[i].first, ver_list[i].second, rid++); std::unique_ptr rowset_writer; ASSERT_TRUE(RowsetFactory::create_rowset_writer(rowset_writer_context, &rowset_writer).ok()); diff --git a/be/test/storage/tablet_updates_test.cpp b/be/test/storage/tablet_updates_test.cpp index b7176856e4ecc..da1226218abff 100644 --- a/be/test/storage/tablet_updates_test.cpp +++ b/be/test/storage/tablet_updates_test.cpp @@ -78,7 +78,7 @@ class TabletUpdatesTest : public testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->thread_safe_get_tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; @@ -90,7 +90,7 @@ class TabletUpdatesTest : public testing::Test { if (empty) { return *writer->build(); } - auto schema = ChunkHelper::convert_schema(tablet->tablet_schema()); + auto schema = ChunkHelper::convert_schema(tablet->thread_safe_get_tablet_schema()); auto chunk = ChunkHelper::new_chunk(schema, keys.size()); auto& cols = chunk->columns(); for (int64_t key : keys) { @@ -133,7 +133,7 @@ class TabletUpdatesTest : public testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->thread_safe_get_tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = OVERLAP_UNKNOWN; @@ -145,7 +145,7 @@ class TabletUpdatesTest : public testing::Test { if (empty) { return *writer->build(); } - auto schema = ChunkHelper::convert_schema(tablet->tablet_schema()); + auto schema = ChunkHelper::convert_schema(tablet->thread_safe_get_tablet_schema()); for (int i = 0; i < keys_by_segment.size(); i++) { auto chunk = ChunkHelper::new_chunk(schema, keys_by_segment[i].size()); auto& cols = chunk->columns(); @@ -192,13 +192,13 @@ class TabletUpdatesTest : public testing::Test { writer_context.rowset_state = COMMITTED; writer_context.partial_update_tablet_schema = partial_schema; writer_context.referenced_column_ids = column_indexes; - writer_context.tablet_schema = partial_schema.get(); + writer_context.tablet_schema = partial_schema; writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; std::unique_ptr writer; EXPECT_TRUE(RowsetFactory::create_rowset_writer(writer_context, &writer).ok()); - auto schema = ChunkHelper::convert_schema(*partial_schema.get()); + auto schema = ChunkHelper::convert_schema(partial_schema); if (keys.size() > 0) { auto chunk = ChunkHelper::new_chunk(schema, keys.size()); @@ -225,13 +225,13 @@ class TabletUpdatesTest : public testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->thread_safe_get_tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; std::unique_ptr writer; EXPECT_TRUE(RowsetFactory::create_rowset_writer(writer_context, &writer).ok()); - auto schema = ChunkHelper::convert_schema(tablet->tablet_schema()); + auto schema = ChunkHelper::convert_schema(tablet->thread_safe_get_tablet_schema()); for (std::size_t written_rows = 0; written_rows < keys.size(); written_rows += max_rows_per_segment) { auto chunk = ChunkHelper::new_chunk(schema, max_rows_per_segment); auto& cols = chunk->columns(); @@ -254,13 +254,13 @@ class TabletUpdatesTest : public testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->thread_safe_get_tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; std::unique_ptr writer; EXPECT_TRUE(RowsetFactory::create_rowset_writer(writer_context, &writer).ok()); - auto schema = ChunkHelper::convert_schema(tablet->tablet_schema()); + auto schema = ChunkHelper::convert_schema(tablet->thread_safe_get_tablet_schema()); const auto nkeys = keys.size(); auto chunk = ChunkHelper::new_chunk(schema, nkeys); auto& cols = chunk->columns(); @@ -283,13 +283,13 @@ class TabletUpdatesTest : public testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->thread_safe_get_tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; std::unique_ptr writer; EXPECT_TRUE(RowsetFactory::create_rowset_writer(writer_context, &writer).ok()); - auto schema = ChunkHelper::convert_schema(tablet->tablet_schema()); + auto schema = ChunkHelper::convert_schema(tablet->thread_safe_get_tablet_schema()); const auto nkeys = keys.size(); auto chunk = ChunkHelper::new_chunk(schema, nkeys); auto& cols = chunk->columns(); @@ -312,13 +312,13 @@ class TabletUpdatesTest : public testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->thread_safe_get_tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; std::unique_ptr writer; EXPECT_TRUE(RowsetFactory::create_rowset_writer(writer_context, &writer).ok()); - auto schema = ChunkHelper::convert_schema(tablet->tablet_schema()); + auto schema = ChunkHelper::convert_schema(tablet->thread_safe_get_tablet_schema()); const auto keys_size = all_cols[0].size(); auto chunk = ChunkHelper::new_chunk(schema, keys_size); auto& cols = chunk->columns(); @@ -813,7 +813,7 @@ static ssize_t read_until_eof(const ChunkIteratorPtr& iter) { } static Status read_with_cancel(const TabletSharedPtr& tablet, int64_t version) { - Schema schema = ChunkHelper::convert_schema(tablet->tablet_schema()); + Schema schema = ChunkHelper::convert_schema(tablet->thread_safe_get_tablet_schema()); TabletReader reader(tablet, Version(0, version), schema); TabletReaderParams params; RuntimeState state; @@ -842,7 +842,7 @@ static Status read_with_cancel(const TabletSharedPtr& tablet, int64_t version) { } static ssize_t read_tablet(const TabletSharedPtr& tablet, int64_t version) { - Schema schema = ChunkHelper::convert_schema(tablet->tablet_schema()); + Schema schema = ChunkHelper::convert_schema(tablet->thread_safe_get_tablet_schema()); TabletReader reader(tablet, Version(0, version), schema); auto iter = create_tablet_iterator(reader, schema); if (iter == nullptr) { @@ -852,7 +852,7 @@ static ssize_t read_tablet(const TabletSharedPtr& tablet, int64_t version) { } static ssize_t read_tablet_and_compare(const TabletSharedPtr& tablet, int64_t version, const vector& keys) { - Schema schema = ChunkHelper::convert_schema(tablet->tablet_schema()); + Schema schema = ChunkHelper::convert_schema(tablet->thread_safe_get_tablet_schema()); TabletReader reader(tablet, Version(0, version), schema); auto iter = create_tablet_iterator(reader, schema); if (iter == nullptr) { @@ -863,7 +863,7 @@ static ssize_t read_tablet_and_compare(const TabletSharedPtr& tablet, int64_t ve static ssize_t read_tablet_and_compare_schema_changed(const TabletSharedPtr& tablet, int64_t version, const vector& keys) { - Schema schema = ChunkHelper::convert_schema(tablet->tablet_schema()); + Schema schema = ChunkHelper::convert_schema(tablet->thread_safe_get_tablet_schema()); TabletReader reader(tablet, Version(0, version), schema); auto iter = create_tablet_iterator(reader, schema); if (iter == nullptr) { @@ -898,7 +898,7 @@ static ssize_t read_tablet_and_compare_schema_changed(const TabletSharedPtr& tab static ssize_t read_tablet_and_compare_schema_changed_sort_key1(const TabletSharedPtr& tablet, int64_t version, const vector& keys) { - Schema schema = ChunkHelper::convert_schema(tablet->tablet_schema()); + Schema schema = ChunkHelper::convert_schema(tablet->thread_safe_get_tablet_schema()); TabletReader reader(tablet, Version(0, version), schema); auto iter = create_tablet_iterator(reader, schema); if (iter == nullptr) { @@ -933,7 +933,7 @@ static ssize_t read_tablet_and_compare_schema_changed_sort_key1(const TabletShar static ssize_t read_tablet_and_compare_schema_changed_sort_key2(const TabletSharedPtr& tablet, int64_t version, const vector& keys) { - Schema schema = ChunkHelper::convert_schema(tablet->tablet_schema()); + Schema schema = ChunkHelper::convert_schema(tablet->thread_safe_get_tablet_schema()); TabletReader reader(tablet, Version(0, version), schema); auto iter = create_tablet_iterator(reader, schema); if (iter == nullptr) { @@ -968,7 +968,7 @@ static ssize_t read_tablet_and_compare_schema_changed_sort_key2(const TabletShar static ssize_t read_tablet_and_compare_sort_key_error_encode_case(const TabletSharedPtr& tablet, int64_t version, const vector& keys) { - Schema schema = ChunkHelper::convert_schema(tablet->tablet_schema()); + Schema schema = ChunkHelper::convert_schema(tablet->thread_safe_get_tablet_schema()); TabletReader reader(tablet, Version(0, version), schema); auto iter = create_tablet_iterator(reader, schema); if (iter == nullptr) { @@ -1002,7 +1002,7 @@ static ssize_t read_tablet_and_compare_sort_key_error_encode_case(const TabletSh static ssize_t read_tablet_and_compare_nullable_sort_key(const TabletSharedPtr& tablet, int64_t version, const vector>& all_cols) { - Schema schema = ChunkHelper::convert_schema(tablet->tablet_schema()); + Schema schema = ChunkHelper::convert_schema(tablet->thread_safe_get_tablet_schema()); TabletReader reader(tablet, Version(0, version), schema); auto iter = create_tablet_iterator(reader, schema); if (iter == nullptr) { @@ -1407,7 +1407,7 @@ void TabletUpdatesTest::test_remove_expired_versions(bool enable_persistent_inde ASSERT_EQ(0, read_tablet(_tablet, 1)); // Create iterators before remove expired version, but read them after removal. - Schema schema = ChunkHelper::convert_schema(_tablet->tablet_schema()); + Schema schema = ChunkHelper::convert_schema(_tablet->thread_safe_get_tablet_schema()); TabletReader reader1(_tablet, Version(0, 1), schema); TabletReader reader2(_tablet, Version(0, 2), schema); TabletReader reader3(_tablet, Version(0, 3), schema); @@ -1746,7 +1746,7 @@ TEST_F(TabletUpdatesTest, horizontal_compaction_with_sort_key) { // the time interval is not enough after last compaction EXPECT_EQ(best_tablet->updates()->get_compaction_score(), -1); - auto schema = ChunkHelper::convert_schema(_tablet->tablet_schema()); + auto schema = ChunkHelper::convert_schema(_tablet->thread_safe_get_tablet_schema()); auto sk_chunk = ChunkHelper::new_chunk(schema, loop); auto& cols = sk_chunk->columns(); for (int i = 0; i < loop; i++) { @@ -1762,7 +1762,7 @@ TEST_F(TabletUpdatesTest, horizontal_compaction_with_sort_key) { ASSERT_TRUE(rowset->get_segment_sk_index(&sk_index_values).ok()); } ASSERT_EQ(sk_index_values.size(), loop); - size_t keys = _tablet->tablet_schema().num_short_key_columns(); + size_t keys = _tablet->thread_safe_get_tablet_schema()->num_short_key_columns(); for (size_t i = 0; i < loop; i++) { SeekTuple tuple(schema, sk_chunk->get(i).datums()); std::string encoded_key = tuple.short_key_encode(keys, {1, 2}, 0); @@ -1963,7 +1963,7 @@ TEST_F(TabletUpdatesTest, vertical_compaction_with_sort_key) { // the time interval is not enough after last compaction EXPECT_EQ(best_tablet->updates()->get_compaction_score(), -1); - auto schema = ChunkHelper::convert_schema(_tablet->tablet_schema()); + auto schema = ChunkHelper::convert_schema(_tablet->thread_safe_get_tablet_schema()); auto sk_chunk = ChunkHelper::new_chunk(schema, loop); auto& cols = sk_chunk->columns(); for (int i = 0; i < loop; i++) { @@ -1979,7 +1979,7 @@ TEST_F(TabletUpdatesTest, vertical_compaction_with_sort_key) { ASSERT_TRUE(rowset->get_segment_sk_index(&sk_index_values).ok()); } ASSERT_EQ(sk_index_values.size(), loop); - size_t keys = _tablet->tablet_schema().num_short_key_columns(); + size_t keys = _tablet->thread_safe_get_tablet_schema()->num_short_key_columns(); for (size_t i = 0; i < loop; i++) { SeekTuple tuple(schema, sk_chunk->get(i).datums()); std::string encoded_key = tuple.short_key_encode(keys, {1, 2}, 0); @@ -2162,8 +2162,8 @@ void TabletUpdatesTest::test_convert_from(bool enable_persistent_index) { tablet_to_schema_change->set_tablet_state(TABLET_NOTREADY); auto chunk_changer = std::make_unique(tablet_to_schema_change->tablet_schema()); - for (int i = 0; i < tablet_to_schema_change->tablet_schema().num_columns(); ++i) { - const auto& new_column = tablet_to_schema_change->tablet_schema().column(i); + for (int i = 0; i < tablet_to_schema_change->unsafe_tablet_schema_ref().num_columns(); ++i) { + const auto& new_column = tablet_to_schema_change->unsafe_tablet_schema_ref().column(i); int32_t column_index = _tablet->field_index(std::string{new_column.name()}); auto column_mapping = chunk_changer->get_mutable_column_mapping(i); if (column_index >= 0) { @@ -2197,8 +2197,8 @@ void TabletUpdatesTest::test_convert_from_with_pending(bool enable_persistent_in tablet_to_schema_change->set_tablet_state(TABLET_NOTREADY); auto chunk_changer = std::make_unique(tablet_to_schema_change->tablet_schema()); - for (int i = 0; i < tablet_to_schema_change->tablet_schema().num_columns(); ++i) { - const auto& new_column = tablet_to_schema_change->tablet_schema().column(i); + for (int i = 0; i < tablet_to_schema_change->unsafe_tablet_schema_ref().num_columns(); ++i) { + const auto& new_column = tablet_to_schema_change->unsafe_tablet_schema_ref().column(i); int32_t column_index = _tablet->field_index(std::string{new_column.name()}); auto column_mapping = chunk_changer->get_mutable_column_mapping(i); if (column_index >= 0) { @@ -2234,8 +2234,8 @@ void TabletUpdatesTest::test_convert_from_with_mutiple_segment(bool enable_persi tablet_to_schema_change->set_tablet_state(TABLET_NOTREADY); auto chunk_changer = std::make_unique(tablet_to_schema_change->tablet_schema()); - for (int i = 0; i < tablet_to_schema_change->tablet_schema().num_columns(); ++i) { - const auto& new_column = tablet_to_schema_change->tablet_schema().column(i); + for (int i = 0; i < tablet_to_schema_change->unsafe_tablet_schema_ref().num_columns(); ++i) { + const auto& new_column = tablet_to_schema_change->unsafe_tablet_schema_ref().column(i); int32_t column_index = _tablet->field_index(std::string{new_column.name()}); auto column_mapping = chunk_changer->get_mutable_column_mapping(i); if (column_index >= 0) { @@ -2301,8 +2301,8 @@ void TabletUpdatesTest::test_reorder_from(bool enable_persistent_index) { tablet_with_sort_key1->set_tablet_state(TABLET_NOTREADY); auto chunk_changer = std::make_unique(tablet_with_sort_key1->tablet_schema()); - for (int i = 0; i < tablet_with_sort_key1->tablet_schema().num_columns(); ++i) { - const auto& new_column = tablet_with_sort_key1->tablet_schema().column(i); + for (int i = 0; i < tablet_with_sort_key1->unsafe_tablet_schema_ref().num_columns(); ++i) { + const auto& new_column = tablet_with_sort_key1->unsafe_tablet_schema_ref().column(i); int32_t column_index = _tablet->field_index(std::string{new_column.name()}); auto column_mapping = chunk_changer->get_mutable_column_mapping(i); if (column_index >= 0) { @@ -2318,8 +2318,8 @@ void TabletUpdatesTest::test_reorder_from(bool enable_persistent_index) { const auto& tablet_with_sort_key2 = create_tablet_with_sort_key(rand(), rand(), {2}); tablet_with_sort_key2->set_tablet_state(TABLET_NOTREADY); chunk_changer = std::make_unique(tablet_with_sort_key2->tablet_schema()); - for (int i = 0; i < tablet_with_sort_key2->tablet_schema().num_columns(); ++i) { - const auto& new_column = tablet_with_sort_key2->tablet_schema().column(i); + for (int i = 0; i < tablet_with_sort_key2->unsafe_tablet_schema_ref().num_columns(); ++i) { + const auto& new_column = tablet_with_sort_key2->unsafe_tablet_schema_ref().column(i); int32_t column_index = _tablet->field_index(std::string{new_column.name()}); auto column_mapping = chunk_changer->get_mutable_column_mapping(i); if (column_index >= 0) { @@ -3334,7 +3334,7 @@ void TabletUpdatesTest::test_get_column_values(bool enable_persistent_index) { ASSERT_TRUE(tablet->rowset_commit(3, create_rowsets(tablet, keys, max_rows_per_segment)).ok()); std::vector read_column_ids = {1, 2}; std::vector> read_columns(read_column_ids.size()); - const auto& tablet_schema = tablet->tablet_schema(); + const auto& tablet_schema = tablet->unsafe_tablet_schema_ref(); for (auto i = 0; i < read_column_ids.size(); i++) { const auto read_column_id = read_column_ids[i]; auto tablet_column = tablet_schema.column(read_column_id); @@ -3563,7 +3563,7 @@ TEST_F(TabletUpdatesTest, multiple_delete_and_upsert) { writer_context.partition_id = 0; writer_context.rowset_path_prefix = _tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &_tablet->tablet_schema(); + writer_context.tablet_schema = _tablet->thread_safe_get_tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; @@ -3576,7 +3576,7 @@ TEST_F(TabletUpdatesTest, multiple_delete_and_upsert) { for (int i = 0; i < 100; i++) { keys.emplace_back(i); } - auto schema = ChunkHelper::convert_schema(_tablet->tablet_schema()); + auto schema = ChunkHelper::convert_schema(_tablet->thread_safe_get_tablet_schema()); auto chunk = ChunkHelper::new_chunk(schema, keys.size()); auto& cols = chunk->columns(); for (int64_t key : keys) { @@ -3592,7 +3592,7 @@ TEST_F(TabletUpdatesTest, multiple_delete_and_upsert) { for (int64_t i = 0; i < 50; i++) { deletes.append_datum(Datum(i)); } - auto schema = ChunkHelper::convert_schema(_tablet->tablet_schema()); + auto schema = ChunkHelper::convert_schema(_tablet->thread_safe_get_tablet_schema()); auto chunk = ChunkHelper::new_chunk(schema, 0); CHECK_OK(writer->flush_chunk_with_deletes(*chunk, deletes)); } @@ -3602,7 +3602,7 @@ TEST_F(TabletUpdatesTest, multiple_delete_and_upsert) { for (int i = 0; i < 50; i++) { keys.emplace_back(i); } - auto schema = ChunkHelper::convert_schema(_tablet->tablet_schema()); + auto schema = ChunkHelper::convert_schema(_tablet->thread_safe_get_tablet_schema()); auto chunk = ChunkHelper::new_chunk(schema, keys.size()); auto& cols = chunk->columns(); for (int64_t key : keys) { @@ -3624,7 +3624,7 @@ TEST_F(TabletUpdatesTest, multiple_delete_and_upsert) { deletes.append_datum(Datum(i)); } - auto schema = ChunkHelper::convert_schema(_tablet->tablet_schema()); + auto schema = ChunkHelper::convert_schema(_tablet->thread_safe_get_tablet_schema()); auto chunk = ChunkHelper::new_chunk(schema, keys.size()); auto& cols = chunk->columns(); for (int64_t key : keys) { @@ -3640,14 +3640,14 @@ TEST_F(TabletUpdatesTest, multiple_delete_and_upsert) { for (int64_t i = 150; i < 200; i++) { deletes.append_datum(Datum(i)); } - auto schema = ChunkHelper::convert_schema(_tablet->tablet_schema()); + auto schema = ChunkHelper::convert_schema(_tablet->thread_safe_get_tablet_schema()); auto chunk = ChunkHelper::new_chunk(schema, 0); CHECK_OK(writer->flush_chunk_with_deletes(*chunk, deletes)); } RowsetSharedPtr rowset = *writer->build(); ASSERT_TRUE(_tablet->rowset_commit(2, rowset).ok()); - Schema schema = ChunkHelper::convert_schema(_tablet->tablet_schema()); + Schema schema = ChunkHelper::convert_schema(_tablet->thread_safe_get_tablet_schema()); TabletReader reader(_tablet, Version(0, 2), schema); auto iter = create_tablet_iterator(reader, schema); ASSERT_TRUE(iter != nullptr); diff --git a/be/test/storage/task/engine_storage_migration_task_test.cpp b/be/test/storage/task/engine_storage_migration_task_test.cpp index 7f58e7c6e55af..e7ca5977e9217 100644 --- a/be/test/storage/task/engine_storage_migration_task_test.cpp +++ b/be/test/storage/task/engine_storage_migration_task_test.cpp @@ -94,7 +94,7 @@ class EngineStorageMigrationTaskTest : public testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &tablet->tablet_schema(); + writer_context.tablet_schema = tablet->tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; @@ -141,11 +141,11 @@ class EngineStorageMigrationTaskTest : public testing::Test { TabletManager* tablet_manager = starrocks::StorageEngine::instance()->tablet_manager(); TabletSharedPtr tablet = tablet_manager->get_tablet(12345); ASSERT_TRUE(tablet != nullptr); - const TabletSchema& tablet_schema = tablet->tablet_schema(); + const TabletSchemaCSPtr& tablet_schema = tablet->tablet_schema(); // create rowset RowsetWriterContext rowset_writer_context; - create_rowset_writer_context(&rowset_writer_context, tablet->schema_hash_path(), &tablet_schema); + create_rowset_writer_context(&rowset_writer_context, tablet->schema_hash_path(), tablet_schema); std::unique_ptr rowset_writer; ASSERT_TRUE(RowsetFactory::create_rowset_writer(rowset_writer_context, &rowset_writer).ok()); @@ -217,7 +217,8 @@ class EngineStorageMigrationTaskTest : public testing::Test { } static void create_rowset_writer_context(RowsetWriterContext* rowset_writer_context, - const std::string& schema_hash_path, const TabletSchema* tablet_schema) { + const std::string& schema_hash_path, + const TabletSchemaCSPtr& tablet_schema) { RowsetId rowset_id; rowset_id.init(10000); rowset_writer_context->rowset_id = rowset_id; @@ -231,7 +232,7 @@ class EngineStorageMigrationTaskTest : public testing::Test { rowset_writer_context->version.second = 2; } - static void rowset_writer_add_rows(std::unique_ptr& writer, const TabletSchema& tablet_schema) { + static void rowset_writer_add_rows(std::unique_ptr& writer, const TabletSchemaCSPtr& tablet_schema) { std::vector test_data; auto schema = ChunkHelper::convert_schema(tablet_schema); auto chunk = ChunkHelper::new_chunk(schema, 1024); diff --git a/be/test/storage/update_manager_test.cpp b/be/test/storage/update_manager_test.cpp index 40d22b397e627..8549a6516b7fa 100644 --- a/be/test/storage/update_manager_test.cpp +++ b/be/test/storage/update_manager_test.cpp @@ -58,7 +58,7 @@ class UpdateManagerTest : public testing::Test { writer_context.partition_id = 0; writer_context.rowset_path_prefix = _tablet->schema_hash_path(); writer_context.rowset_state = COMMITTED; - writer_context.tablet_schema = &_tablet->tablet_schema(); + writer_context.tablet_schema = _tablet->tablet_schema(); writer_context.version.first = 0; writer_context.version.second = 0; writer_context.segments_overlap = NONOVERLAPPING; diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobV2.java index 55d00dbb14a1c..268f91fc275fd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobV2.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/AlterJobV2.java @@ -134,6 +134,10 @@ public JobState getJobState() { return jobState; } + public void setJobState(JobState jobState) { + this.jobState = jobState; + } + public JobType getType() { return type; } @@ -166,6 +170,10 @@ public long getFinishedTimeMs() { return finishedTimeMs; } + public void setFinishedTimeMs(long finishedTimeMs) { + this.finishedTimeMs = finishedTimeMs; + } + /** * The keyword 'synchronized' only protects 2 methods: * run() and cancel() diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/com/starrocks/alter/MaterializedViewHandler.java index 703e5f71514c0..fb2775fabea78 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/MaterializedViewHandler.java @@ -531,6 +531,9 @@ private List checkAndPrepareMaterializedView(CreateMaterializedViewStmt newMVColumns.add(mvColumnItem.toMVColumn(olapTable)); } } + + // set MV column unique id to Column.COLUMN_UNIQUE_ID_INIT_VALUE support old unique id rule. + newMVColumns.forEach(column -> column.setUniqueId(Column.COLUMN_UNIQUE_ID_INIT_VALUE)); return newMVColumns; } diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/RollupJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/RollupJobV2.java index b89efb48cabc2..6b1e2a943877d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/RollupJobV2.java @@ -858,10 +858,6 @@ public Map getPartitionIdToRollupIndex() { return partitionIdToRollupIndex; } - public void setJobState(JobState jobState) { - this.jobState = jobState; - } - private void setColumnsDefineExpr(Map columnNameToDefineExpr) { for (Map.Entry entry : columnNameToDefineExpr.entrySet()) { for (Column column : rollupSchema) { diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java index c14617cddb30d..f7bcb02948cc9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeHandler.java @@ -50,6 +50,7 @@ import com.starrocks.catalog.AggregateType; import com.starrocks.catalog.Column; import com.starrocks.catalog.Database; +import com.starrocks.catalog.DistributionInfo; import com.starrocks.catalog.DistributionInfo.DistributionInfoType; import com.starrocks.catalog.DynamicPartitionProperty; import com.starrocks.catalog.ForeignKeyConstraint; @@ -63,6 +64,8 @@ import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.OlapTable.OlapTableState; import com.starrocks.catalog.Partition; +import com.starrocks.catalog.PartitionInfo; +import com.starrocks.catalog.PartitionType; import com.starrocks.catalog.PhysicalPartition; import com.starrocks.catalog.RangePartitionInfo; import com.starrocks.catalog.Replica; @@ -78,12 +81,15 @@ import com.starrocks.common.ErrorReport; import com.starrocks.common.FeConstants; import com.starrocks.common.MarkedCountDownLatch; +import com.starrocks.common.MetaNotFoundException; +import com.starrocks.common.NotImplementedException; import com.starrocks.common.Pair; import com.starrocks.common.UserException; import com.starrocks.common.util.DynamicPartitionUtil; import com.starrocks.common.util.ListComparator; import com.starrocks.common.util.PropertyAnalyzer; import com.starrocks.common.util.WriteQuorum; +import com.starrocks.persist.TableAddOrDropColumnsInfo; import com.starrocks.qe.ConnectContext; import com.starrocks.qe.ShowResultSet; import com.starrocks.server.GlobalStateMgr; @@ -120,10 +126,12 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.IntSupplier; import java.util.stream.Collectors; import javax.annotation.Nullable; public class SchemaChangeHandler extends AlterHandler { + private static final Logger LOG = LogManager.getLogger(SchemaChangeHandler.class); // all shadow indexes should have this prefix in name @@ -136,8 +144,17 @@ public SchemaChangeHandler() { super("schema change"); } - private void processAddColumn(AddColumnClause alterClause, OlapTable olapTable, - Map> indexSchemaMap) throws DdlException { + /** + * @param alterClause + * @param olapTable + * @param indexSchemaMap + * @param colUniqueIdSupplier for multi add columns clause, we need stash middle state of maxColUniqueId + * @return true: can light schema change, false: cannot light schema change + * @throws DdlException + */ + private boolean processAddColumn(AddColumnClause alterClause, OlapTable olapTable, + Map> indexSchemaMap, + IntSupplier colUniqueIdSupplier) throws DdlException { Column column = alterClause.getColumn(); ColumnPosition columnPos = alterClause.getColPos(); String targetIndexName = alterClause.getRollupName(); @@ -153,13 +170,27 @@ private void processAddColumn(AddColumnClause alterClause, OlapTable olapTable, } Set newColNameSet = Sets.newHashSet(column.getName()); - addColumnInternal(olapTable, column, columnPos, targetIndexId, baseIndexId, - indexSchemaMap, newColNameSet); + // only new table generate ColUniqueId, exist table do not. + if (olapTable.getMaxColUniqueId() > Column.COLUMN_UNIQUE_ID_INIT_VALUE) { + column.setUniqueId(colUniqueIdSupplier.getAsInt()); + } + + return addColumnInternal(olapTable, column, columnPos, targetIndexId, baseIndexId, indexSchemaMap, + newColNameSet); } - private void processAddColumns(AddColumnsClause alterClause, OlapTable olapTable, - Map> indexSchemaMap) throws DdlException { + /** + * @param alterClause + * @param olapTable + * @param indexSchemaMap + * @param colUniqueIdSupplier for multi add columns clause, we need stash middle state of maxColUniqueId + * @return true: can light schema change, false: cannot light schema change + * @throws DdlException + */ + private boolean processAddColumns(AddColumnsClause alterClause, OlapTable olapTable, + Map> indexSchemaMap, + IntSupplier colUniqueIdSupplier) throws DdlException { List columns = alterClause.getColumns(); String targetIndexName = alterClause.getRollupName(); checkIndexExists(olapTable, targetIndexName); @@ -178,23 +209,41 @@ private void processAddColumns(AddColumnsClause alterClause, OlapTable olapTable targetIndexId = olapTable.getIndexIdByName(targetIndexName); } + //for new table calculate column unique id + if (olapTable.getMaxColUniqueId() > Column.COLUMN_UNIQUE_ID_INIT_VALUE) { + for (Column column : columns) { + column.setUniqueId(colUniqueIdSupplier.getAsInt()); + } + } + + boolean ligthSchemaChange = true; if (alterClause.getGeneratedColumnPos() == null) { for (Column column : columns) { - addColumnInternal(olapTable, column, null, targetIndexId, baseIndexId, - indexSchemaMap, newColNameSet); + ligthSchemaChange = addColumnInternal(olapTable, column, null, targetIndexId, baseIndexId, indexSchemaMap, + newColNameSet); } } else { for (int i = columns.size() - 1; i >= 0; --i) { Column column = columns.get(i); - addColumnInternal(olapTable, column, alterClause.getGeneratedColumnPos(), + ligthSchemaChange = addColumnInternal(olapTable, column, alterClause.getGeneratedColumnPos(), targetIndexId, baseIndexId, indexSchemaMap, newColNameSet); } } + return ligthSchemaChange; } - private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable, - Map> indexSchemaMap, List indexes) - throws DdlException { + /** + * @param alterClause + * @param olapTable + * @param indexSchemaMap + * @param indexes + * @return true: can light schema change, false: cannot + * @throws DdlException + */ + private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTable, + Map> indexSchemaMap, List indexes) throws DdlException { + + boolean lightSchemaChange = olapTable.getUseLightSchemaChange(); String dropColName = alterClause.getColName(); String targetIndexName = alterClause.getRollupName(); checkIndexExists(olapTable, targetIndexName); @@ -211,6 +260,8 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable * Can not drop any key column is has value with REPLACE method */ if (KeysType.PRIMARY_KEYS == olapTable.getKeysType()) { + // not support drop column in primary model + lightSchemaChange = false; long baseIndexId = olapTable.getBaseIndexId(); List baseSchema = indexSchemaMap.get(baseIndexId); boolean isKey = baseSchema.stream().anyMatch(c -> c.isKey() && c.getName().equalsIgnoreCase(dropColName)); @@ -229,6 +280,7 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable long baseIndexId = olapTable.getBaseIndexId(); List baseSchema = indexSchemaMap.get(baseIndexId); boolean isKey = baseSchema.stream().anyMatch(c -> c.isKey() && c.getName().equalsIgnoreCase(dropColName)); + lightSchemaChange &= !isKey; if (isKey) { throw new DdlException("Can not drop key column in Unique data model table"); } @@ -238,6 +290,7 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable long baseIndexId = olapTable.getBaseIndexId(); List baseSchema = indexSchemaMap.get(baseIndexId); boolean isKey = baseSchema.stream().anyMatch(c -> c.isKey() && c.getName().equalsIgnoreCase(dropColName)); + lightSchemaChange &= !isKey; boolean hasReplaceColumn = baseSchema.stream().map(Column::getAggregationType) .anyMatch(agg -> agg == AggregateType.REPLACE || agg == AggregateType.REPLACE_IF_NOT_NULL); if (isKey && hasReplaceColumn) { @@ -248,6 +301,7 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable long targetIndexId = olapTable.getIndexIdByName(targetIndexName); List targetIndexSchema = indexSchemaMap.get(targetIndexId); boolean isKey = targetIndexSchema.stream().anyMatch(c -> c.isKey() && c.getName().equalsIgnoreCase(dropColName)); + lightSchemaChange &= !isKey; boolean hasReplaceColumn = targetIndexSchema.stream().map(Column::getAggregationType) .anyMatch(agg -> agg == AggregateType.REPLACE || agg == AggregateType.REPLACE_IF_NOT_NULL); if (isKey && hasReplaceColumn) { @@ -255,6 +309,15 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable "Can not drop key column when rollup has value column with REPLACE aggregation method"); } } + } else if (KeysType.DUP_KEYS == olapTable.getKeysType()) { + long baseIndexId = olapTable.getBaseIndexId(); + List baseSchema = indexSchemaMap.get(baseIndexId); + for (Column column : baseSchema) { + if (column.isKey() && column.getName().equalsIgnoreCase(dropColName)) { + lightSchemaChange = false; + break; + } + } } // Remove all Index that contains a column with the name dropColName. @@ -263,26 +326,64 @@ private void processDropColumn(DropColumnClause alterClause, OlapTable olapTable if (targetIndexName == null) { // if not specify rollup index, column should be dropped from both base and rollup indexes. long baseIndexId = olapTable.getBaseIndexId(); - boolean removed = indexSchemaMap.get(baseIndexId).removeIf(c -> c.getName().equalsIgnoreCase(dropColName)); + LinkedList columns = indexSchemaMap.get(baseIndexId); + Iterator columnIterator = columns.iterator(); + boolean removed = false; + while (columnIterator.hasNext()) { + Column column = columnIterator.next(); + if (column.getName().equalsIgnoreCase(dropColName)) { + columnIterator.remove(); + removed = true; + if (column.isKey()) { + lightSchemaChange = false; + } + } + } + if (!removed) { throw new DdlException("Column does not exists: " + dropColName); } + for (Long indexId : olapTable.getIndexIdListExceptBaseIndex()) { - indexSchemaMap.get(indexId).removeIf(c -> c.getName().equalsIgnoreCase(dropColName)); + columns = indexSchemaMap.get(indexId); + columnIterator = columns.iterator(); + while (columnIterator.hasNext()) { + Column column = columnIterator.next(); + if (column.getName().equalsIgnoreCase(dropColName)) { + columnIterator.remove(); + if (column.isKey()) { + lightSchemaChange = false; + } + } + } } } else { // if specify rollup index, only drop column from specified rollup index long targetIndexId = olapTable.getIndexIdByName(targetIndexName); - boolean removed = indexSchemaMap.get(targetIndexId).removeIf(c -> c.getName().equalsIgnoreCase(dropColName)); + LinkedList columns = indexSchemaMap.get(targetIndexId); + Iterator columnIterator = columns.iterator(); + boolean removed = false; + while (columnIterator.hasNext()) { + Column column = columnIterator.next(); + if (column.getName().equalsIgnoreCase(dropColName)) { + columnIterator.remove(); + removed = true; + if (column.isKey()) { + lightSchemaChange = false; + } + } + } + if (!removed) { throw new DdlException("Column does not exists: " + dropColName); } } + return lightSchemaChange; } // User can modify column type and column position private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapTable, - Map> indexSchemaMap) throws DdlException { + Map> indexSchemaMap) throws DdlException { Column modColumn = alterClause.getColumn(); if (KeysType.PRIMARY_KEYS == olapTable.getKeysType()) { if (olapTable.getBaseColumn(modColumn.getName()).isKey()) { @@ -412,6 +513,7 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT // retain old column name modColumn.setName(oriColumn.getName()); + modColumn.setUniqueId(oriColumn.getUniqueId()); if (!oriColumn.isGeneratedColumn() && modColumn.isGeneratedColumn()) { throw new DdlException("Can not modify a non-generated column to a generated column"); @@ -549,7 +651,7 @@ private void processModifyColumn(ModifyColumnClause alterClause, OlapTable olapT } private void processReorderColumn(ReorderColumnsClause alterClause, OlapTable olapTable, - Map> indexSchemaMap) throws DdlException { + Map> indexSchemaMap) throws DdlException { List orderedColNames = alterClause.getColumnsByPos(); String targetIndexName = alterClause.getRollupName(); checkIndexExists(olapTable, targetIndexName); @@ -586,7 +688,7 @@ private void processReorderColumn(ReorderColumnsClause alterClause, OlapTable ol } private void processReorderColumnOfPrimaryKey(ReorderColumnsClause alterClause, OlapTable olapTable, - Map> indexSchemaMap, List sortKeyIdxes) + Map> indexSchemaMap, List sortKeyIdxes) throws DdlException { LinkedList targetIndexSchema = indexSchemaMap.get(olapTable.getIndexIdByName(olapTable.getName())); // check sort key column list @@ -610,20 +712,35 @@ private void processReorderColumnOfPrimaryKey(ReorderColumnsClause alterClause, } } - /* - * Add 'newColumn' to specified index. - * Modified schema will be saved in 'indexSchemaMap' + /** + * @param olapTable + * @param newColumn Add 'newColumn' to specified index. + * @param columnPos + * @param targetIndexId + * @param baseIndexId + * @param indexSchemaMap Modified schema will be saved in 'indexSchemaMap' + * @param newColNameSet + * @return true: can light schema change, false: cannot + * @throws DdlException */ - private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosition columnPos, - long targetIndexId, long baseIndexId, - Map> indexSchemaMap, - Set newColNameSet) throws DdlException { + private boolean addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosition columnPos, + long targetIndexId, long baseIndexId, + Map> indexSchemaMap, + Set newColNameSet) throws DdlException { Column.DefaultValueType defaultValueType = newColumn.getDefaultValueType(); if (defaultValueType == Column.DefaultValueType.VARY) { throw new DdlException("unsupported default expr:" + newColumn.getDefaultExpr().getExpr()); } + + //only new table generate ColUniqueId, exist table do not. + boolean lightSchemaChange = olapTable.getMaxColUniqueId() > Column.COLUMN_UNIQUE_ID_INIT_VALUE; + String newColName = newColumn.getName(); + //make sure olapTable has locked + LOG.debug("table: {}, newColumn: {}, uniqueId: {}", olapTable.getName(), newColumn.getName(), + newColumn.getUniqueId()); + // check the validation of aggregation method on column. // also fill the default aggregation method if not specified. if (KeysType.PRIMARY_KEYS == olapTable.getKeysType()) { @@ -647,7 +764,7 @@ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosi } newColumn.setIsKey(true); } else if (newColumn.getAggregationType() == AggregateType.SUM - && newColumn.getDefaultValue() != null && !newColumn.getDefaultValue().equals("0")) { + && newColumn.getDefaultValue() != null && !"0".equals(newColumn.getDefaultValue())) { throw new DdlException( "The default value of '" + newColName + "' with SUM aggregation function must be zero"); } @@ -685,11 +802,18 @@ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosi throw new DdlException("PERCENTILE_UNION must be used in AGG_KEYS"); } + //type key column do not allow light schema change. + if (newColumn.isKey()) { + lightSchemaChange = false; + } + // check if the new column already exist in base schema. // do not support adding new column which already exist in base schema. - boolean found = olapTable.getBaseSchema().stream().anyMatch(c -> c.getName().equalsIgnoreCase(newColName)); - if (found) { - throw new DdlException("Can not add column which already exists in base table: " + newColName); + Optional foundColumn = olapTable.getBaseSchema().stream() + .filter(c -> c.getName().equalsIgnoreCase(newColName)).findFirst(); + if (foundColumn.isPresent() && newColumn.equals(foundColumn.get())) { + throw new DdlException( + "Can not add column which already exists in base table: " + newColName); } /* @@ -710,6 +834,7 @@ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosi if (KeysType.PRIMARY_KEYS == olapTable.getKeysType()) { List modIndexSchema = indexSchemaMap.get(baseIndexId); checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true); + lightSchemaChange = false; if (targetIndexId != -1L) { throw new DdlException("Can not add column: " + newColName + " to rollup index"); } @@ -728,9 +853,10 @@ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosi modIndexSchema = indexSchemaMap.get(baseIndexId); checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true); if (targetIndexId == -1L) { - return; + return lightSchemaChange; } // 2. add to rollup + lightSchemaChange = false; modIndexSchema = indexSchemaMap.get(targetIndexId); checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false); } @@ -740,9 +866,10 @@ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosi List modIndexSchema = indexSchemaMap.get(baseIndexId); checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, true); // no specified target index. return - return; + return lightSchemaChange; } else { // add to rollup index + lightSchemaChange = false; List modIndexSchema = indexSchemaMap.get(targetIndexId); checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false); @@ -766,13 +893,15 @@ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosi if (targetIndexId == -1L) { // no specified target index. return - return; + return lightSchemaChange; } + lightSchemaChange = false; // 2. add to rollup index modIndexSchema = indexSchemaMap.get(targetIndexId); checkAndAddColumn(modIndexSchema, newColumn, columnPos, newColNameSet, false); } + return lightSchemaChange; } /* @@ -785,7 +914,7 @@ private void addColumnInternal(OlapTable olapTable, Column newColumn, ColumnPosi * So that k1 will be added to base index 'twice', and we just ignore this repeat adding. */ private void checkAndAddColumn(List modIndexSchema, Column newColumn, ColumnPosition columnPos, - Set newColNameSet, boolean isBaseIndex) throws DdlException { + Set newColNameSet, boolean isBaseIndex) throws DdlException { int posIndex = -1; String newColName = newColumn.getName(); boolean hasPos = (columnPos != null && !columnPos.isFirst()); @@ -861,7 +990,7 @@ private void checkAssignedTargetIndexName(String baseIndexName, String targetInd } private AlterJobV2 createJob(long dbId, OlapTable olapTable, Map> indexSchemaMap, - Map propertyMap, List indexes) throws UserException { + Map propertyMap, List indexes) throws UserException { if (olapTable.getState() == OlapTableState.ROLLUP) { throw new DdlException("Table[" + olapTable.getName() + "]'s is doing ROLLUP job"); } @@ -1102,7 +1231,7 @@ private AlterJobV2 createJob(long dbId, OlapTable olapTable, Map originSortKeyIdxes = index.getSortKeyIdxes(); for (Integer colIdx : originSortKeyIdxes) { String columnName = index.getSchema().get(colIdx).getName(); - Optional oneCol = + Optional oneCol = alterSchema.stream().filter(c -> c.getName().equalsIgnoreCase(columnName)).findFirst(); if (!oneCol.isPresent()) { LOG.warn("Sort Key Column[" + columnName + "] not exists in new schema"); @@ -1114,20 +1243,20 @@ private AlterJobV2 createJob(long dbId, OlapTable olapTable, Map> indexSchemaMap, - List sortKeyIdxes) throws UserException { + Map> indexSchemaMap, + List sortKeyIdxes) throws UserException { if (olapTable.getState() == OlapTableState.ROLLUP) { throw new DdlException("Table[" + olapTable.getName() + "]'s is doing ROLLUP job"); } @@ -1200,7 +1329,7 @@ public List> getAlterJobInfosByDb(Database db) { } private void getAlterJobV2Infos(Database db, List alterJobsV2, - List> schemaChangeJobInfos) { + List> schemaChangeJobInfos) { ConnectContext ctx = ConnectContext.get(); for (AlterJobV2 alterJob : alterJobsV2) { if (alterJob.getDbId() != db.getId()) { @@ -1228,6 +1357,19 @@ public Optional getActiveTxnIdOfTable(long tableId) { @VisibleForTesting @Nullable public AlterJobV2 analyzeAndCreateJob(List alterClauses, Database db, OlapTable olapTable) throws UserException { + + //alterClauses can or cannot light schema change + boolean lightSchemaChange = true; + //for multi add colmuns clauses + IntSupplier colUniqueIdSupplier = new IntSupplier() { + private int pendingMaxColUniqueId = olapTable.getMaxColUniqueId(); + + @Override + public int getAsInt() { + pendingMaxColUniqueId++; + return pendingMaxColUniqueId; + } + }; // index id -> index schema Map> indexSchemaMap = new HashMap<>(); for (Map.Entry> entry : olapTable.getIndexIdToSchema().entrySet()) { @@ -1308,18 +1450,25 @@ public AlterJobV2 analyzeAndCreateJob(List alterClauses, Database d if (alterClause instanceof AddColumnClause) { // add column - processAddColumn((AddColumnClause) alterClause, olapTable, indexSchemaMap); + lightSchemaChange = + processAddColumn((AddColumnClause) alterClause, olapTable, indexSchemaMap, + colUniqueIdSupplier); } else if (alterClause instanceof AddColumnsClause) { // add columns - processAddColumns((AddColumnsClause) alterClause, olapTable, indexSchemaMap); + lightSchemaChange = + processAddColumns((AddColumnsClause) alterClause, olapTable, indexSchemaMap, colUniqueIdSupplier); } else if (alterClause instanceof DropColumnClause) { // drop column and drop indexes on this column - processDropColumn((DropColumnClause) alterClause, olapTable, indexSchemaMap, newIndexes); + lightSchemaChange = + processDropColumn((DropColumnClause) alterClause, olapTable, indexSchemaMap, + newIndexes); } else if (alterClause instanceof ModifyColumnClause) { // modify column processModifyColumn((ModifyColumnClause) alterClause, olapTable, indexSchemaMap); + lightSchemaChange = false; } else if (alterClause instanceof ReorderColumnsClause) { // reorder column + lightSchemaChange = false; if (olapTable.getKeysType() == KeysType.PRIMARY_KEYS) { List sortKeyIdxes = new ArrayList<>(); processReorderColumnOfPrimaryKey((ReorderColumnsClause) alterClause, olapTable, indexSchemaMap, sortKeyIdxes); @@ -1331,16 +1480,29 @@ public AlterJobV2 analyzeAndCreateJob(List alterClauses, Database d } else if (alterClause instanceof ModifyTablePropertiesClause) { // modify table properties // do nothing, properties are already in propertyMap + lightSchemaChange = false; } else if (alterClause instanceof CreateIndexClause) { + lightSchemaChange = false; processAddIndex((CreateIndexClause) alterClause, olapTable, newIndexes); } else if (alterClause instanceof DropIndexClause) { + lightSchemaChange = false; processDropIndex((DropIndexClause) alterClause, olapTable, newIndexes); } else { Preconditions.checkState(false); } } // end for alter clauses - return createJob(db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes); + LOG.debug("processAddColumns, table: {}({}), ligthSchemaChange: {}", olapTable.getName(), olapTable.getId(), + lightSchemaChange); + + if (lightSchemaChange) { + long jobId = GlobalStateMgr.getCurrentState().getNextId(); + //for schema change add/drop value column optimize, direct modify table meta. + modifyTableAddOrDropColumns(db, olapTable, indexSchemaMap, newIndexes, jobId, false); + return null; + } else { + return createJob(db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes); + } } @Override @@ -1390,7 +1552,7 @@ private void sendClearAlterTask(Database db, OlapTable olapTable) { } public void updateTableMeta(Database db, String tableName, Map properties, - TTabletMetaType metaType) + TTabletMetaType metaType) throws DdlException { List partitions = Lists.newArrayList(); OlapTable olapTable; @@ -1448,7 +1610,7 @@ public void updateTableMeta(Database db, String tableName, Map p // and as long as the modification of metadata is successful, // the final consistency will be achieved through the report handler public boolean updateBinlogConfigMeta(Database db, Long tableId, Map properties, - TTabletMetaType metaType) { + TTabletMetaType metaType) { List partitions = Lists.newArrayList(); OlapTable olapTable; BinlogConfig newBinlogConfig; @@ -1555,9 +1717,9 @@ public boolean updateBinlogConfigMeta(Database db, Long tableId, Map partitionNames, - Map properties) throws DdlException { + String tableName, + List partitionNames, + Map properties) throws DdlException { OlapTable olapTable; db.readLock(); try { @@ -1595,10 +1757,10 @@ public void updatePartitionsInMemoryMeta(Database db, * This operation may return partial successfully, with a exception to inform user to retry */ public void updateBinlogPartitionTabletMeta(Database db, - String tableName, - String partitionName, - BinlogConfig binlogConfig, - TTabletMetaType metaType) throws DdlException { + String tableName, + String partitionName, + BinlogConfig binlogConfig, + TTabletMetaType metaType) throws DdlException { // be id -> Map>> beIdToTabletIdWithHash = Maps.newHashMap(); db.readLock(); @@ -1678,10 +1840,10 @@ public void updateBinlogPartitionTabletMeta(Database db, * This operation may return partial successfully, with a exception to inform user to retry */ public void updatePartitionTabletMeta(Database db, - String tableName, - String partitionName, - boolean metaValue, - TTabletMetaType metaType) throws DdlException { + String tableName, + String partitionName, + boolean metaValue, + TTabletMetaType metaType) throws DdlException { // be id -> Map>> beIdToTabletIdWithHash = Maps.newHashMap(); db.readLock(); @@ -1938,4 +2100,181 @@ private void processDropIndex(DropIndexClause alterClause, OlapTable olapTable, } } } + + // the invoker should keep write lock + public void modifyTableAddOrDropColumns(Database db, OlapTable olapTable, + Map> indexSchemaMap, + List indexes, long jobId, boolean isReplay) + throws DdlException, NotImplementedException { + db.writeLock(); + try { + LOG.debug("indexSchemaMap:{}, indexes:{}", indexSchemaMap, indexes); + if (olapTable.getState() == OlapTableState.ROLLUP) { + throw new DdlException("Table[" + olapTable.getName() + "]'s is doing ROLLUP job"); + } + + // for now table's state can only be NORMAL + Preconditions.checkState(olapTable.getState() == OlapTableState.NORMAL, olapTable.getState().name()); + + // for bitmapIndex + boolean hasIndexChange = false; + Set newSet = new HashSet<>(indexes); + Set oriSet = new HashSet<>(olapTable.getIndexes()); + if (!newSet.equals(oriSet)) { + hasIndexChange = true; + } + + // begin checking each table + // ATTN: DO NOT change any meta in this loop + Map> changedIndexIdToSchema = Maps.newHashMap(); + for (Long alterIndexId : indexSchemaMap.keySet()) { + // Must get all columns including invisible columns. + // Because in alter process, all columns must be considered. + List alterSchema = indexSchemaMap.get(alterIndexId); + + LOG.debug("index[{}] is changed. start checking...", alterIndexId); + // 1. check order: a) has key; b) value after key + boolean meetValue = false; + boolean hasKey = false; + for (Column column : alterSchema) { + if (column.isKey() && meetValue) { + throw new DdlException( + "Invalid column order. value should be after key. index[" + olapTable.getIndexNameById( + alterIndexId) + "]"); + } + if (!column.isKey()) { + meetValue = true; + } else { + hasKey = true; + } + } + if (!hasKey) { + throw new DdlException("No key column left. index[" + olapTable.getIndexNameById(alterIndexId) + "]"); + } + + // 2. check partition key + PartitionInfo partitionInfo = olapTable.getPartitionInfo(); + if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { + List partitionColumns = partitionInfo.getPartitionColumns(); + for (Column partitionCol : partitionColumns) { + boolean found = false; + for (Column alterColumn : alterSchema) { + if (alterColumn.nameEquals(partitionCol.getName(), true)) { + found = true; + break; + } + } // end for alterColumns + + if (!found && alterIndexId == olapTable.getBaseIndexId()) { + // 2.1 partition column cannot be deleted. + throw new DdlException( + "Partition column[" + partitionCol.getName() + "] cannot be dropped. index[" + + olapTable.getIndexNameById(alterIndexId) + "]"); + } + } // end for partitionColumns + } + + // 3. check distribution key: + DistributionInfo distributionInfo = olapTable.getDefaultDistributionInfo(); + if (distributionInfo.getType() == DistributionInfoType.HASH) { + List distributionColumns = ((HashDistributionInfo) distributionInfo).getDistributionColumns(); + for (Column distributionCol : distributionColumns) { + boolean found = false; + for (Column alterColumn : alterSchema) { + if (alterColumn.nameEquals(distributionCol.getName(), true)) { + found = true; + break; + } + } // end for alterColumns + if (!found && alterIndexId == olapTable.getBaseIndexId()) { + // 2.2 distribution column cannot be deleted. + throw new DdlException( + "Distribution column[" + distributionCol.getName() + "] cannot be dropped. index[" + + olapTable.getIndexNameById(alterIndexId) + "]"); + } + } // end for distributionCols + } + + // 5. store the changed columns for edit log + changedIndexIdToSchema.put(alterIndexId, alterSchema); + + LOG.debug("schema change[{}-{}-{}] check pass.", db.getId(), olapTable.getId(), alterIndexId); + } // end for indices + + if (changedIndexIdToSchema.isEmpty() && !hasIndexChange) { + throw new DdlException("Nothing is changed. please check your alter stmt."); + } + + // update base index schema + long baseIndexId = olapTable.getBaseIndexId(); + List indexIds = new ArrayList(); + indexIds.add(baseIndexId); + indexIds.addAll(olapTable.getIndexIdListExceptBaseIndex()); + for (int i = 0; i < indexIds.size(); i++) { + List indexSchema = indexSchemaMap.get(indexIds.get(i)); + MaterializedIndexMeta currentIndexMeta = olapTable.getIndexMetaByIndexId(indexIds.get(i)); + currentIndexMeta.setSchema(indexSchema); + + int currentSchemaVersion = currentIndexMeta.getSchemaVersion(); + int newSchemaVersion = currentSchemaVersion + 1; + currentIndexMeta.setSchemaVersion(newSchemaVersion); + } + olapTable.setIndexes(indexes); + olapTable.rebuildFullSchema(); + + // update max column unique id + int maxColUniqueId = olapTable.getMaxColUniqueId(); + for (Column column : indexSchemaMap.get(olapTable.getBaseIndexId())) { + if (column.getUniqueId() > maxColUniqueId) { + maxColUniqueId = column.getUniqueId(); + } + } + olapTable.setMaxColUniqueId(maxColUniqueId); + + if (!isReplay) { + TableAddOrDropColumnsInfo info = new TableAddOrDropColumnsInfo(db.getId(), olapTable.getId(), + indexSchemaMap, indexes, jobId); + LOG.debug("logModifyTableAddOrDropColumns info:{}", info); + GlobalStateMgr.getCurrentState().getEditLog().logModifyTableAddOrDropColumns(info); + } + + SchemaChangeJobV2 schemaChangeJob = new SchemaChangeJobV2(jobId, db.getId(), olapTable.getId(), + olapTable.getName(), 1000); + schemaChangeJob.setJobState(AlterJobV2.JobState.FINISHED); + schemaChangeJob.setFinishedTimeMs(System.currentTimeMillis()); + this.addAlterJobV2(schemaChangeJob); + + LOG.info("finished modify table's add or drop columns. table: {}, is replay: {}", olapTable.getName(), + isReplay); + } finally { + db.writeUnlock(); + } + } + + public void replayModifyTableAddOrDropColumns(TableAddOrDropColumnsInfo info) throws + MetaNotFoundException { + LOG.debug("info:{}", info); + long dbId = info.getDbId(); + long tableId = info.getTableId(); + Map> indexSchemaMap = info.getIndexSchemaMap(); + List indexes = info.getIndexes(); + long jobId = info.getJobId(); + + Database db = GlobalStateMgr.getCurrentState().getDb(dbId); + Table table = db.getTable(tableId); + Preconditions.checkArgument(table instanceof OlapTable, + "Target of light schema change must be olap table"); + OlapTable olapTable = (OlapTable) table; + try { + db.writeLock(); + modifyTableAddOrDropColumns(db, olapTable, indexSchemaMap, indexes, jobId, true); + } catch (DdlException e) { + // should not happen + LOG.warn("failed to replay modify table add or drop columns", e); + } catch (NotImplementedException e) { + LOG.error("InternalError", e); + } finally { + db.writeUnlock(); + } + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java index 9625b4164c97e..aa8ae5c91cc76 100644 --- a/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/com/starrocks/alter/SchemaChangeJobV2.java @@ -623,6 +623,7 @@ protected void runWaitingTxnJob() throws AlterCancelException { } int shadowSchemaHash = indexSchemaVersionAndHashMap.get(shadowIdxId).schemaHash; int originSchemaHash = tbl.getSchemaHashByIndexId(indexIdMap.get(shadowIdxId)); + List originSchemaColumns = tbl.getSchemaByIndexId(originIdxId); for (Tablet shadowTablet : shadowIdx.getTablets()) { long shadowTabletId = shadowTablet.getId(); @@ -631,7 +632,8 @@ protected void runWaitingTxnJob() throws AlterCancelException { AlterReplicaTask rollupTask = AlterReplicaTask.alterLocalTablet( shadowReplica.getBackendId(), dbId, tableId, partitionId, shadowIdxId, shadowTabletId, originTabletId, shadowReplica.getId(), - shadowSchemaHash, originSchemaHash, visibleVersion, jobId, generatedColumnReq); + shadowSchemaHash, originSchemaHash, visibleVersion, jobId, + generatedColumnReq, originSchemaColumns); schemaChangeBatchTask.addTask(rollupTask); } } @@ -767,7 +769,7 @@ protected void runRunningJob() throws AlterCancelException { db.writeUnlock(); } - editLog.waitInfinity(start, future); + EditLog.waitInfinity(start, future); LOG.info("schema change job finished: {}", jobId); this.span.end(); @@ -932,6 +934,17 @@ private void onFinished(OlapTable tbl) { tbl.setIndexes(indexes); } + //update max column unique id + int maxColUniqueId = tbl.getMaxColUniqueId(); + for (Column column : tbl.getFullSchema()) { + if (column.getUniqueId() > maxColUniqueId) { + maxColUniqueId = column.getUniqueId(); + } + } + tbl.setMaxColUniqueId(maxColUniqueId); + LOG.debug("fullSchema:{}, maxColUniqueId:{}", tbl.getFullSchema(), maxColUniqueId); + + tbl.setState(OlapTableState.NORMAL); tbl.lastSchemaUpdateTime.set(System.currentTimeMillis()); } diff --git a/fe/fe-core/src/main/java/com/starrocks/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/com/starrocks/analysis/SlotDescriptor.java index c187e533b8d63..5f947779e318f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/analysis/SlotDescriptor.java +++ b/fe/fe-core/src/main/java/com/starrocks/analysis/SlotDescriptor.java @@ -47,7 +47,12 @@ import java.util.Collections; import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + public class SlotDescriptor { + + private static final Logger LOG = LogManager.getLogger(SlotDescriptor.class); private final SlotId id; private final TupleDescriptor parent; private Type type; @@ -253,6 +258,10 @@ public TSlotDescriptor toThrift() { } else { type = type.isNull() ? ScalarType.BOOLEAN : type; tSlotDescriptor.setSlotType(type.toThrift()); + if (column != null) { + LOG.debug("column name:{}, column unique id:{}", column.getName(), column.getUniqueId()); + tSlotDescriptor.setCol_unique_id(column.getUniqueId()); + } } tSlotDescriptor.setColumnPos(-1); tSlotDescriptor.setByteOffset(-1); diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/Column.java b/fe/fe-core/src/main/java/com/starrocks/catalog/Column.java index 954563306087f..3c2953946bda8 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/Column.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/Column.java @@ -39,6 +39,7 @@ import com.google.gson.annotations.SerializedName; import com.starrocks.alter.SchemaChangeHandler; import com.starrocks.analysis.Expr; +import com.starrocks.analysis.IndexDef; import com.starrocks.analysis.NullLiteral; import com.starrocks.analysis.SlotRef; import com.starrocks.analysis.StringLiteral; @@ -74,6 +75,8 @@ public class Column implements Writable, GsonPreProcessable, GsonPostProcessable { public static final String CAN_NOT_CHANGE_DEFAULT_VALUE = "Can not change default value"; + public static final int COLUMN_UNIQUE_ID_INIT_VALUE = -1; + @SerializedName(value = "name") private String name; @@ -109,6 +112,9 @@ public class Column implements Writable, GsonPreProcessable, GsonPostProcessable // Currently, analyzed define expr is only used when creating materialized views, so the define expr in RollupJob must be analyzed. // In other cases, such as define expr in `MaterializedIndexMeta`, it may not be analyzed after being relayed. private Expr defineExpr; // use to define column in materialize view + @SerializedName(value = "uniqueId") + private int uniqueId; + @SerializedName(value = "materializedColumnExpr") private GsonUtils.ExpressionSerializedObject generatedColumnExprSerialized; private Expr generatedColumnExpr; @@ -119,32 +125,41 @@ public Column() { this.isAggregationTypeImplicit = false; this.isKey = false; this.stats = new ColumnStats(); + this.uniqueId = -1; } public Column(String name, Type dataType) { - this(name, dataType, false, null, false, null, ""); + this(name, dataType, false, null, false, null, "", COLUMN_UNIQUE_ID_INIT_VALUE); Preconditions.checkArgument(dataType.isValid()); } public Column(String name, Type dataType, boolean isAllowNull) { - this(name, dataType, false, null, isAllowNull, null, ""); + this(name, dataType, false, null, isAllowNull, null, "", COLUMN_UNIQUE_ID_INIT_VALUE); Preconditions.checkArgument(dataType.isValid()); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, String defaultValue, String comment) { this(name, type, isKey, aggregateType, false, - new ColumnDef.DefaultValueDef(true, new StringLiteral(defaultValue)), comment); + new ColumnDef.DefaultValueDef(true, new StringLiteral(defaultValue)), comment, + COLUMN_UNIQUE_ID_INIT_VALUE); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, ColumnDef.DefaultValueDef defaultValue, String comment) { - this(name, type, isKey, aggregateType, false, defaultValue, comment); + this(name, type, isKey, aggregateType, false, defaultValue, comment, + COLUMN_UNIQUE_ID_INIT_VALUE); + } + + public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, + ColumnDef.DefaultValueDef defaultValueDef, String comment) { + this(name, type, isKey, aggregateType, isAllowNull, defaultValueDef, comment, + COLUMN_UNIQUE_ID_INIT_VALUE); } public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, - ColumnDef.DefaultValueDef defaultValueDef, String comment) { + ColumnDef.DefaultValueDef defaultValueDef, String comment, int columnUniqId) { this.name = name; if (this.name == null) { this.name = ""; @@ -175,6 +190,7 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType this.comment = comment; this.stats = new ColumnStats(); this.generatedColumnExpr = null; + this.uniqueId = columnUniqId; } public Column(Column column) { @@ -191,6 +207,7 @@ public Column(Column column) { this.defaultExpr = column.defaultExpr; Preconditions.checkArgument(this.type.isComplexType() || this.type.getPrimitiveType() != PrimitiveType.INVALID_TYPE); + this.uniqueId = column.getUniqueId(); } public void setName(String newName) { @@ -347,6 +364,8 @@ public TColumn toThrift() { // scalar type or nested type // If this field is set, column_type will be ignored. tColumn.setType_desc(type.toThrift()); + tColumn.setCol_unique_id(uniqueId); + return tColumn; } @@ -714,4 +733,24 @@ public void gsonPreProcess() throws IOException { generatedColumnExprSerialized = new GsonUtils.ExpressionSerializedObject(generatedColumnExpr.toSql()); } } + + + public void setUniqueId(int colUniqueId) { + this.uniqueId = colUniqueId; + } + + public int getUniqueId() { + return this.uniqueId; + } + + public void setIndexFlag(TColumn tColumn, List indexes) { + for (Index index : indexes) { + if (index.getIndexType() == IndexDef.IndexType.BITMAP) { + List columns = index.getColumns(); + if (tColumn.getColumn_name().equals(columns.get(0))) { + tColumn.setHas_bitmap_index(true); + } + } + } + } } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedIndexMeta.java b/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedIndexMeta.java index cd92973c53aad..dfed0ccde4076 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedIndexMeta.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/MaterializedIndexMeta.java @@ -200,6 +200,14 @@ public int hashCode() { return Long.hashCode(indexId); } + public void setSchema(List newSchema) { + this.schema = newSchema; + } + + public void setSchemaVersion(int newSchemaVersion) { + this.schemaVersion = newSchemaVersion; + } + @Override public boolean equals(Object obj) { if (!(obj instanceof MaterializedIndexMeta)) { diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java index 5217a483fc53d..99ede28263bbb 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/OlapTable.java @@ -214,6 +214,9 @@ public enum OlapTableState { @SerializedName(value = "tableProperty") protected TableProperty tableProperty; + @SerializedName(value = "maxColUniqueId") + protected int maxColUniqueId = -1; + protected BinlogConfig curBinlogConfig; // After ensuring that all binlog config of tablets in BE have taken effect, @@ -368,6 +371,20 @@ public TableProperty getTableProperty() { return this.tableProperty; } + public int incAndGetMaxColUniqueId() { + this.maxColUniqueId++; + return this.maxColUniqueId; + } + + public int getMaxColUniqueId() { + return this.maxColUniqueId; + } + + public void setMaxColUniqueId(int maxColUniqueId) { + this.maxColUniqueId = maxColUniqueId; + } + + public boolean dynamicPartitionExists() { return tableProperty != null && tableProperty.getDynamicPartitionProperty() != null @@ -1649,6 +1666,7 @@ public void write(DataOutput out) throws IOException { tableProperty.write(out); } tempPartitions.write(out); + out.writeInt(maxColUniqueId); } @Override @@ -1749,6 +1767,7 @@ public void readFields(DataInput in) throws IOException { } tempPartitions.unsetPartitionInfo(); + maxColUniqueId = in.readInt(); // In the present, the fullSchema could be rebuilt by schema change while the // properties is changed by MV. // After that, some properties of fullSchema and nameToColumn may be not same as @@ -2548,6 +2567,24 @@ public void setForeignKeyConstraints(List foreignKeyConstr tableProperty.setForeignKeyConstraints(foreignKeyConstraints); } + public Boolean getUseLightSchemaChange() { + if (tableProperty != null) { + return tableProperty.getUseSchemaLightChange(); + } + // property is set false by default + return false; + } + + public void setUseLightSchemaChange(boolean useLightSchemaChange) { + if (tableProperty == null) { + tableProperty = new TableProperty(new HashMap<>()); + } + tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_USE_LIGHT_SCHEMA_CHANGE, + Boolean.valueOf(useLightSchemaChange).toString()); + tableProperty.buildUseLightSchemaChange(); + } + + @Override public void onReload() { analyzePartitionInfo(); diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java b/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java index 55173769ba3a6..3fa2cc2bd705c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/TableProperty.java @@ -198,6 +198,8 @@ public static String valueList() { // foreign key constraint for mv rewrite private List foreignKeyConstraints; + private Boolean useSchemaLightChange; + private PeriodDuration dataCachePartitionDuration; public TableProperty(Map properties) { @@ -687,6 +689,16 @@ public void clearBinlogAvailableVersion() { } } + public TableProperty buildUseLightSchemaChange() { + useSchemaLightChange = Boolean.parseBoolean( + properties.getOrDefault(PropertyAnalyzer.PROPERTIES_USE_LIGHT_SCHEMA_CHANGE, "false")); + return this; + } + + public Boolean getUseSchemaLightChange() { + return useSchemaLightChange; + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); @@ -718,5 +730,6 @@ public void gsonPostProcess() throws IOException { buildBinlogAvailableVersion(); buildConstraint(); buildDataCachePartitionDuration(); + buildUseLightSchemaChange(); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java index 88c16aab4eb34..0b63b221007be 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/TabletInvertedIndex.java @@ -702,6 +702,10 @@ public void deleteTablet(long tabletId) { } } + public Table getReplicaMetaTable() { + return replicaMetaTable; + } + public void addReplica(long tabletId, Replica replica) { if (GlobalStateMgr.isCheckpointThread()) { return; diff --git a/fe/fe-core/src/main/java/com/starrocks/common/Config.java b/fe/fe-core/src/main/java/com/starrocks/common/Config.java index 95b91ae39c511..9537ed33550a4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/Config.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/Config.java @@ -143,6 +143,9 @@ public class Config extends ConfigBase { @ConfField(mutable = true) public static long slow_lock_log_every_ms = 3000L; + @ConfField + public static String custom_config_dir = "/conf"; + /** * dump_log_dir: * This specifies FE dump log dir. @@ -2450,6 +2453,9 @@ public class Config extends ConfigBase { @ConfField(mutable = true) public static int external_table_commit_timeout_ms = 10000; // 10s + @ConfField(mutable = true) + public static boolean allow_default_light_schema_change = false; + @ConfField(mutable = false) public static int pipe_listener_interval_millis = 1000; @ConfField(mutable = false) diff --git a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java index 049b702735468..3a48ee42ef90e 100644 --- a/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/common/util/PropertyAnalyzer.java @@ -58,6 +58,7 @@ import com.starrocks.catalog.Type; import com.starrocks.catalog.UniqueConstraint; import com.starrocks.common.AnalysisException; +import com.starrocks.common.Config; import com.starrocks.common.Pair; import com.starrocks.lake.DataCacheInfo; import com.starrocks.qe.ConnectContext; @@ -175,6 +176,9 @@ public class PropertyAnalyzer { // positive value: use [0, mv_randomize_start) as random interval public static final String PROPERTY_MV_RANDOMIZE_START = "mv_randomize_start"; + // light schema change + public static final String PROPERTIES_USE_LIGHT_SCHEMA_CHANGE = "light_schema_change"; + public static final String PROPERTIES_DEFAULT_PREFIX = "default."; public static DataProperty analyzeDataProperty(Map properties, @@ -536,6 +540,30 @@ public static int analyzeSchemaVersion(Map properties) throws An return schemaVersion; } + public static Boolean analyzeUseLightSchemaChange(Map properties) throws AnalysisException { + if (properties == null || properties.isEmpty()) { + return false; + } + String value = properties.get(PROPERTIES_USE_LIGHT_SCHEMA_CHANGE); + // set light schema change false by default + if (Config.allow_default_light_schema_change) { + properties.remove(PROPERTIES_USE_LIGHT_SCHEMA_CHANGE); + return true; + } + if (null == value) { + return false; + } + properties.remove(PROPERTIES_USE_LIGHT_SCHEMA_CHANGE); + if (Boolean.TRUE.toString().equalsIgnoreCase(value)) { + return true; + } else if (Boolean.FALSE.toString().equalsIgnoreCase(value)) { + return false; + } + throw new AnalysisException(PROPERTIES_USE_LIGHT_SCHEMA_CHANGE + + " must be `true` or `false`"); + } + + public static Set analyzeBloomFilterColumns(Map properties, List columns, boolean isPrimaryKey) throws AnalysisException { Set bfColumns = null; diff --git a/fe/fe-core/src/main/java/com/starrocks/journal/JournalEntity.java b/fe/fe-core/src/main/java/com/starrocks/journal/JournalEntity.java index b3346aa7170d8..a1995d94e5be4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/com/starrocks/journal/JournalEntity.java @@ -118,6 +118,7 @@ import com.starrocks.persist.SetReplicaStatusOperationLog; import com.starrocks.persist.ShardInfo; import com.starrocks.persist.SwapTableOperationLog; +import com.starrocks.persist.TableAddOrDropColumnsInfo; import com.starrocks.persist.TableInfo; import com.starrocks.persist.TablePropertyInfo; import com.starrocks.persist.TransactionIdInfo; @@ -1019,6 +1020,10 @@ public void readFields(DataInput in) throws IOException { data = MVEpoch.read(in); isRead = true; break; + case OperationType.OP_MODIFY_TABLE_ADD_OR_DROP_COLUMNS: + data = TableAddOrDropColumnsInfo.read(in); + isRead = true; + break; case OperationType.OP_SET_DEFAULT_STORAGE_VOLUME: data = SetDefaultStorageVolumeLog.read(in); isRead = true; diff --git a/fe/fe-core/src/main/java/com/starrocks/load/OlapDeleteJob.java b/fe/fe-core/src/main/java/com/starrocks/load/OlapDeleteJob.java index 599d7b222361e..e19339b1500e9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/OlapDeleteJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/OlapDeleteJob.java @@ -40,6 +40,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.starrocks.analysis.Predicate; +import com.starrocks.catalog.Column; import com.starrocks.catalog.Database; import com.starrocks.catalog.LocalTablet; import com.starrocks.catalog.MaterializedIndex; @@ -63,6 +64,7 @@ import com.starrocks.task.AgentTaskExecutor; import com.starrocks.task.AgentTaskQueue; import com.starrocks.task.PushTask; +import com.starrocks.thrift.TColumn; import com.starrocks.thrift.TPriority; import com.starrocks.thrift.TPushType; import com.starrocks.thrift.TTaskType; @@ -72,6 +74,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; @@ -133,6 +136,11 @@ public void run(DeleteStmt stmt, Database db, Table table, List parti long indexId = index.getId(); int schemaHash = olapTable.getSchemaHashByIndexId(indexId); + List columnsDesc = new ArrayList<>(); + for (Column column : olapTable.getSchemaByIndexId(indexId)) { + columnsDesc.add(column.toThrift()); + } + for (Tablet tablet : index.getTablets()) { long tabletId = tablet.getId(); @@ -155,7 +163,7 @@ public void run(DeleteStmt stmt, Database db, Table table, List parti TTaskType.REALTIME_PUSH, getTransactionId(), GlobalStateMgr.getCurrentGlobalTransactionMgr().getTransactionIDGenerator() - .getNextTransactionId()); + .getNextTransactionId(), columnsDesc); pushTask.setIsSchemaChanging(false); pushTask.setCountDownLatch(countDownLatch); diff --git a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadJob.java b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadJob.java index 0950d7d0bc9fe..ed3b56d1580e7 100644 --- a/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadJob.java +++ b/fe/fe-core/src/main/java/com/starrocks/load/loadv2/SparkLoadJob.java @@ -97,6 +97,7 @@ import com.starrocks.thrift.TBrokerRangeDesc; import com.starrocks.thrift.TBrokerScanRange; import com.starrocks.thrift.TBrokerScanRangeParams; +import com.starrocks.thrift.TColumn; import com.starrocks.thrift.TDescriptorTable; import com.starrocks.thrift.TFileFormatType; import com.starrocks.thrift.TFileType; @@ -121,6 +122,7 @@ import java.io.DataOutput; import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -504,6 +506,11 @@ private Set submitPushTasks() throws UserException { long indexId = index.getId(); int schemaHash = indexToSchemaHash.get(indexId); + List columnsDesc = new ArrayList(); + for (Column column : table.getSchemaByIndexId(indexId)) { + columnsDesc.add(column.toThrift()); + } + int bucket = 0; for (Tablet tablet : index.getTablets()) { long tabletId = tablet.getId(); @@ -520,12 +527,13 @@ private Set submitPushTasks() throws UserException { long replicaId = replica.getId(); tabletAllReplicas.add(replicaId); long backendId = replica.getBackendId(); - Backend backend = GlobalStateMgr.getCurrentState().getCurrentSystemInfo() + Backend backend = GlobalStateMgr.getCurrentSystemInfo() .getBackend(backendId); pushTask(backendId, tableId, partitionId, indexId, tabletId, replicaId, schemaHash, params, batchTask, tabletMetaStr, - backend, replica, tabletFinishedReplicas, TTabletType.TABLET_TYPE_DISK); + backend, replica, tabletFinishedReplicas, + TTabletType.TABLET_TYPE_DISK, columnsDesc); } if (tabletAllReplicas.size() == 0) { @@ -554,7 +562,7 @@ private Set submitPushTasks() throws UserException { pushTask(backend.getId(), tableId, partitionId, indexId, tabletId, tabletId, schemaHash, params, batchTask, tabletMetaStr, backend, new Replica(tabletId, backendId, -1, NORMAL), - tabletFinishedReplicas, TTabletType.TABLET_TYPE_LAKE); + tabletFinishedReplicas, TTabletType.TABLET_TYPE_LAKE, columnsDesc); if (tabletFinishedReplicas.contains(tabletId)) { quorumTablets.add(tabletId); @@ -594,7 +602,7 @@ private void pushTask(long backendId, long tableId, long partitionId, long index AgentBatchTask batchTask, String tabletMetaStr, ComputeNode backend, Replica replica, Set tabletFinishedReplicas, - TTabletType tabletType) + TTabletType tabletType, List columnDesc) throws AnalysisException { if (!tabletToSentReplicaPushTask.containsKey(tabletId) @@ -639,7 +647,7 @@ private void pushTask(long backendId, long tableId, long partitionId, long index 0, id, TPushType.LOAD_V2, TPriority.NORMAL, transactionId, taskSignature, tBrokerScanRange, params.tDescriptorTable, - timezone, tabletType); + timezone, tabletType, columnDesc); if (AgentTaskQueue.addTask(pushTask)) { batchTask.addTask(pushTask); if (!tabletToSentReplicaPushTask.containsKey(tabletId)) { diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java b/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java index 1969b4a701616..f9a9336dc87b9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java +++ b/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java @@ -1053,6 +1053,11 @@ public static void loadJournal(GlobalStateMgr globalStateMgr, JournalEntity jour MaterializedViewMgr.getInstance().replayEpoch(epoch); break; } + case OperationType.OP_MODIFY_TABLE_ADD_OR_DROP_COLUMNS: { + final TableAddOrDropColumnsInfo info = (TableAddOrDropColumnsInfo) journal.getData(); + globalStateMgr.getSchemaChangeHandler().replayModifyTableAddOrDropColumns(info); + break; + } case OperationType.OP_SET_DEFAULT_STORAGE_VOLUME: { SetDefaultStorageVolumeLog log = (SetDefaultStorageVolumeLog) journal.getData(); globalStateMgr.getStorageVolumeMgr().replaySetDefaultStorageVolume(log); @@ -2030,6 +2035,10 @@ private void logJsonObject(short op, Object obj) { logEdit(op, out -> Text.writeString(out, GsonUtils.GSON.toJson(obj))); } + public void logModifyTableAddOrDropColumns(TableAddOrDropColumnsInfo info) { + logEdit(OperationType.OP_MODIFY_TABLE_ADD_OR_DROP_COLUMNS, info); + } + public void logAlterTask(Task changedTask) { logEdit(OperationType.OP_ALTER_TASK, changedTask); } diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java b/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java index 2975798de75d0..00fcdb9be0f7a 100644 --- a/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java +++ b/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java @@ -225,6 +225,9 @@ public class OperationType { public static final short OP_SAVE_AUTO_INCREMENT_ID = 105; public static final short OP_DELETE_AUTO_INCREMENT_ID = 106; + // light schema change for add and drop columns + public static final short OP_MODIFY_TABLE_ADD_OR_DROP_COLUMNS = 107; + // routine load 110~120 @Deprecated public static final short OP_ROUTINE_LOAD_JOB = 110; diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/TableAddOrDropColumnsInfo.java b/fe/fe-core/src/main/java/com/starrocks/persist/TableAddOrDropColumnsInfo.java new file mode 100644 index 0000000000000..554e12edf2c93 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/persist/TableAddOrDropColumnsInfo.java @@ -0,0 +1,139 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file is based on code available under the Apache license here: +// https://github.com/apache/incubator-doris/blob/master/fe/fe-core/src/main/java/org/apache/doris/persist/TableAddOrDropColumnsInfo.java + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +package com.starrocks.persist; + +import com.google.gson.annotations.SerializedName; +import com.starrocks.catalog.Column; +import com.starrocks.catalog.Index; +import com.starrocks.common.io.Text; +import com.starrocks.common.io.Writable; +import com.starrocks.persist.gson.GsonUtils; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * PersistInfo for Table properties + */ +public class TableAddOrDropColumnsInfo implements Writable { + + @SerializedName(value = "dbId") + private long dbId; + @SerializedName(value = "tableId") + private long tableId; + @SerializedName(value = "indexSchemaMap") + private Map> indexSchemaMap; + @SerializedName(value = "indexes") + private List indexes; + @SerializedName(value = "jobId") + private long jobId; + + public TableAddOrDropColumnsInfo(long dbId, long tableId, + Map> indexSchemaMap, List indexes, long jobId) { + this.dbId = dbId; + this.tableId = tableId; + this.indexSchemaMap = indexSchemaMap; + this.indexes = indexes; + this.jobId = jobId; + } + + public long getDbId() { + return dbId; + } + + public long getTableId() { + return tableId; + } + + public Map> getIndexSchemaMap() { + return indexSchemaMap; + } + + public List getIndexes() { + return indexes; + } + + public long getJobId() { + return jobId; + } + + @Override + public void write(DataOutput out) throws IOException { + Text.writeString(out, GsonUtils.GSON.toJson(this)); + } + + public static TableAddOrDropColumnsInfo read(DataInput in) throws IOException { + return GsonUtils.GSON.fromJson(Text.readString(in), TableAddOrDropColumnsInfo.class); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + + if (!(obj instanceof TableAddOrDropColumnsInfo)) { + return false; + } + + TableAddOrDropColumnsInfo info = (TableAddOrDropColumnsInfo) obj; + + return (dbId == info.dbId && tableId == info.tableId + && indexSchemaMap.equals(info.indexSchemaMap) && indexes.equals(info.indexes) + && jobId == info.jobId); + } + + @Override + public int hashCode() { + return Objects.hash(dbId, tableId, indexSchemaMap, indexes, jobId); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(" dbId: ").append(dbId); + sb.append(" tableId: ").append(tableId); + sb.append(" indexSchemaMap: ").append(indexSchemaMap); + sb.append(" indexes: ").append(indexes); + sb.append(" jobId: ").append(jobId); + return sb.toString(); + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java b/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java index e5e0117334117..6755af93868ac 100644 --- a/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/com/starrocks/planner/OlapScanNode.java @@ -78,6 +78,7 @@ import com.starrocks.sql.ast.PartitionNames; import com.starrocks.sql.optimizer.operator.scalar.ColumnRefOperator; import com.starrocks.system.ComputeNode; +import com.starrocks.thrift.TColumn; import com.starrocks.thrift.TExplainLevel; import com.starrocks.thrift.TInternalScanRange; import com.starrocks.thrift.TLakeScanNode; @@ -756,6 +757,8 @@ public int getNumInstances() { protected void toThrift(TPlanNode msg) { List keyColumnNames = new ArrayList(); List keyColumnTypes = new ArrayList(); + List columnsDesc = new ArrayList(); + if (selectedIndexId != -1) { MaterializedIndexMeta indexMeta = olapTable.getIndexMetaByIndexId(selectedIndexId); if (indexMeta != null) { @@ -767,9 +770,14 @@ protected void toThrift(TPlanNode msg) { } } else { for (Column col : olapTable.getSchemaByIndexId(selectedIndexId)) { + TColumn tColumn = col.toThrift(); + col.setIndexFlag(tColumn, olapTable.getIndexes()); + columnsDesc.add(tColumn); + if (!col.isKey()) { - break; + continue; } + keyColumnNames.add(col.getName()); keyColumnTypes.add(col.getPrimitiveType().toThrift()); } @@ -809,7 +817,8 @@ protected void toThrift(TPlanNode msg) { } else { // If you find yourself changing this code block, see also the above code block msg.node_type = TPlanNodeType.OLAP_SCAN_NODE; msg.olap_scan_node = - new TOlapScanNode(desc.getId().asInt(), keyColumnNames, keyColumnTypes, isPreAggregation); + new TOlapScanNode(desc.getId().asInt(), keyColumnNames, + keyColumnTypes, isPreAggregation, columnsDesc); msg.olap_scan_node.setSort_key_column_names(keyColumnNames); msg.olap_scan_node.setRollup_name(olapTable.getIndexNameById(selectedIndexId)); if (!conjuncts.isEmpty()) { diff --git a/fe/fe-core/src/main/java/com/starrocks/planner/OlapTableSink.java b/fe/fe-core/src/main/java/com/starrocks/planner/OlapTableSink.java index d5ca68146de9a..9ad07389544d8 100644 --- a/fe/fe-core/src/main/java/com/starrocks/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/com/starrocks/planner/OlapTableSink.java @@ -76,6 +76,7 @@ import com.starrocks.load.Load; import com.starrocks.server.GlobalStateMgr; import com.starrocks.system.SystemInfoService; +import com.starrocks.thrift.TColumn; import com.starrocks.thrift.TDataSink; import com.starrocks.thrift.TDataSinkType; import com.starrocks.thrift.TExplainLevel; @@ -272,7 +273,7 @@ private TOlapTableSchemaParam createSchema(long dbId, OlapTable table) { TOlapTableSchemaParam schemaParam = new TOlapTableSchemaParam(); schemaParam.setDb_id(dbId); schemaParam.setTable_id(table.getId()); - schemaParam.setVersion(0); + schemaParam.setVersion(table.getIndexMetaByIndexId(table.getBaseIndexId()).getSchemaVersion()); schemaParam.tuple_desc = tupleDescriptor.toThrift(); for (SlotDescriptor slotDesc : tupleDescriptor.getSlots()) { @@ -282,12 +283,18 @@ private TOlapTableSchemaParam createSchema(long dbId, OlapTable table) { for (Map.Entry pair : table.getIndexIdToMeta().entrySet()) { MaterializedIndexMeta indexMeta = pair.getValue(); List columns = Lists.newArrayList(); + List columnsDesc = Lists.newArrayList(); columns.addAll(indexMeta.getSchema().stream().map(Column::getName).collect(Collectors.toList())); + for (Column column : indexMeta.getSchema()) { + TColumn tColumn = column.toThrift(); + column.setIndexFlag(tColumn, table.getIndexes()); + columnsDesc.add(tColumn); + } if (table.getKeysType() == KeysType.PRIMARY_KEYS) { columns.add(Load.LOAD_OP_COLUMN); } TOlapTableIndexSchema indexSchema = new TOlapTableIndexSchema(pair.getKey(), columns, - indexMeta.getSchemaHash()); + indexMeta.getSchemaHash(), columnsDesc); schemaParam.addToIndexes(indexSchema); } return schemaParam; diff --git a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java index d3421a5f351bd..b1b55fb19223d 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java @@ -2728,6 +2728,12 @@ public static void getDdlStmt(String dbName, Table table, List createTab sb.append(WriteQuorum.writeQuorumToName(olapTable.writeQuorum())).append("\""); } + // show lightSchemaChange only when it is set true + if (olapTable.getUseLightSchemaChange()) { + sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_USE_LIGHT_SCHEMA_CHANGE).append("\" = \""); + sb.append(olapTable.getUseLightSchemaChange()).append("\""); + } + // storage media Map properties = olapTable.getTableProperty().getProperties(); diff --git a/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java b/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java index c1fe19d19910c..9e87af56435d6 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/OlapTableFactory.java @@ -75,6 +75,7 @@ import javax.validation.constraints.NotNull; public class OlapTableFactory implements AbstractTableFactory { + private static final Logger LOG = LogManager.getLogger(OlapTableFactory.class); public static final OlapTableFactory INSTANCE = new OlapTableFactory(); @@ -224,6 +225,25 @@ public Table createTable(LocalMetastore metastore, Database db, CreateTableStmt long baseIndexId = metastore.getNextId(); table.setBaseIndexId(baseIndexId); + // get use light schema change + Boolean useLightSchemaChange; + try { + useLightSchemaChange = PropertyAnalyzer.analyzeUseLightSchemaChange(properties); + } catch (AnalysisException e) { + throw new DdlException(e.getMessage()); + } + // only support olap table use light schema change optimization + table.setUseLightSchemaChange(useLightSchemaChange); + if (useLightSchemaChange) { + for (Column column : baseSchema) { + column.setUniqueId(table.incAndGetMaxColUniqueId()); + LOG.debug("table: {}, newColumn: {}, uniqueId: {}", table.getName(), column.getName(), + column.getUniqueId()); + } + } else { + LOG.debug("table: {} doesn't use light schema change", table.getName()); + } + // analyze bloom filter columns Set bfColumns = null; double bfFpp = 0; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ColumnDef.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ColumnDef.java index efe1ab5c94d28..6ae75625a2acd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/ColumnDef.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/ColumnDef.java @@ -510,7 +510,8 @@ public NodePosition getPos() { } public Column toColumn() { - Column col = new Column(name, typeDef.getType(), isKey, aggregateType, isAllowNull, defaultValueDef, comment); + Column col = new Column(name, typeDef.getType(), isKey, aggregateType, isAllowNull, defaultValueDef, comment, + Column.COLUMN_UNIQUE_ID_INIT_VALUE); col.setIsAutoIncrement(isAutoIncrement); col.setGeneratedColumnExpr(generatedColumnExpr); return col; diff --git a/fe/fe-core/src/main/java/com/starrocks/task/AlterReplicaTask.java b/fe/fe-core/src/main/java/com/starrocks/task/AlterReplicaTask.java index ecf504dcf3262..2e7e7471dcc04 100644 --- a/fe/fe-core/src/main/java/com/starrocks/task/AlterReplicaTask.java +++ b/fe/fe-core/src/main/java/com/starrocks/task/AlterReplicaTask.java @@ -39,6 +39,7 @@ import com.starrocks.alter.AlterJobV2; import com.starrocks.analysis.Expr; import com.starrocks.analysis.SlotRef; +import com.starrocks.catalog.Column; import com.starrocks.catalog.Database; import com.starrocks.catalog.LocalTablet; import com.starrocks.catalog.MaterializedIndex; @@ -53,6 +54,7 @@ import com.starrocks.thrift.TAlterMaterializedViewParam; import com.starrocks.thrift.TAlterTabletMaterializedColumnReq; import com.starrocks.thrift.TAlterTabletReqV2; +import com.starrocks.thrift.TColumn; import com.starrocks.thrift.TQueryGlobals; import com.starrocks.thrift.TQueryOptions; import com.starrocks.thrift.TTabletType; @@ -61,6 +63,8 @@ import org.apache.logging.log4j.Logger; import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Map; @@ -72,6 +76,7 @@ * The new replica can be a rollup replica, or a shadow replica of schema change. */ public class AlterReplicaTask extends AgentTask implements Runnable { + private static final Logger LOG = LogManager.getLogger(AlterReplicaTask.class); private final long baseTabletId; @@ -85,36 +90,38 @@ public class AlterReplicaTask extends AgentTask implements Runnable { private final long txnId; private final Map defineExprs; private final TAlterTabletMaterializedColumnReq generatedColumnReq; + private List baseSchemaColumns; public static AlterReplicaTask alterLocalTablet(long backendId, long dbId, long tableId, long partitionId, long rollupIndexId, long rollupTabletId, long baseTabletId, long newReplicaId, int newSchemaHash, int baseSchemaHash, long version, long jobId, - TAlterTabletMaterializedColumnReq generatedColumnReq) { + TAlterTabletMaterializedColumnReq generatedColumnReq, + List baseSchemaColumns) { return new AlterReplicaTask(backendId, dbId, tableId, partitionId, rollupIndexId, rollupTabletId, baseTabletId, newReplicaId, newSchemaHash, baseSchemaHash, version, jobId, AlterJobV2.JobType.SCHEMA_CHANGE, - null, TTabletType.TABLET_TYPE_DISK, 0, generatedColumnReq); + null, TTabletType.TABLET_TYPE_DISK, 0, generatedColumnReq, baseSchemaColumns); } public static AlterReplicaTask alterLakeTablet(long backendId, long dbId, long tableId, long partitionId, long rollupIndexId, - long rollupTabletId, long baseTabletId, long version, long jobId, long txnId) { + long rollupTabletId, long baseTabletId, long version, long jobId, long txnId) { return new AlterReplicaTask(backendId, dbId, tableId, partitionId, rollupIndexId, rollupTabletId, baseTabletId, -1, -1, -1, version, jobId, AlterJobV2.JobType.SCHEMA_CHANGE, - null, TTabletType.TABLET_TYPE_LAKE, txnId, null); + null, TTabletType.TABLET_TYPE_LAKE, txnId, null, Collections.emptyList()); } public static AlterReplicaTask rollupLocalTablet(long backendId, long dbId, long tableId, long partitionId, - long rollupIndexId, long rollupTabletId, long baseTabletId, - long newReplicaId, int newSchemaHash, int baseSchemaHash, long version, - long jobId, Map defineExprs) { + long rollupIndexId, long rollupTabletId, long baseTabletId, + long newReplicaId, int newSchemaHash, int baseSchemaHash, long version, + long jobId, Map defineExprs) { return new AlterReplicaTask(backendId, dbId, tableId, partitionId, rollupIndexId, rollupTabletId, baseTabletId, newReplicaId, newSchemaHash, baseSchemaHash, version, jobId, AlterJobV2.JobType.ROLLUP, - defineExprs, TTabletType.TABLET_TYPE_DISK, 0, null); + defineExprs, TTabletType.TABLET_TYPE_DISK, 0, null, Collections.emptyList()); } private AlterReplicaTask(long backendId, long dbId, long tableId, long partitionId, long rollupIndexId, long rollupTabletId, long baseTabletId, long newReplicaId, int newSchemaHash, int baseSchemaHash, long version, long jobId, AlterJobV2.JobType jobType, Map defineExprs, TTabletType tabletType, - long txnId, TAlterTabletMaterializedColumnReq generatedColumnReq) { + long txnId, TAlterTabletMaterializedColumnReq generatedColumnReq, List baseSchemaColumns) { super(null, backendId, TTaskType.ALTER, dbId, tableId, partitionId, rollupIndexId, rollupTabletId); this.baseTabletId = baseTabletId; @@ -133,6 +140,7 @@ private AlterReplicaTask(long backendId, long dbId, long tableId, long partition this.txnId = txnId; this.generatedColumnReq = generatedColumnReq; + this.baseSchemaColumns = baseSchemaColumns; } public long getBaseTabletId() { @@ -184,7 +192,7 @@ public TAlterTabletReqV2 toThrift() { TQueryGlobals queryGlobals = new TQueryGlobals(); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); queryGlobals.setNow_string(dateFormat.format(new Date())); - queryGlobals.setTimestamp_ms(new Date().getTime()); + queryGlobals.setTimestamp_ms(System.currentTimeMillis()); queryGlobals.setTime_zone(TimeUtils.DEFAULT_TIME_ZONE); TQueryOptions queryOptions = new TQueryOptions(); req.setQuery_globals(queryGlobals); @@ -194,6 +202,14 @@ public TAlterTabletReqV2 toThrift() { req.setTablet_type(tabletType); req.setTxn_id(txnId); req.setJob_id(jobId); + + if (baseSchemaColumns != null) { + List columns = new ArrayList(); + for (Column column : baseSchemaColumns) { + columns.add(column.toThrift()); + } + req.setColumns(columns); + } return req; } @@ -215,7 +231,7 @@ public TAlterTabletReqV2 toThrift() { * So the replica's version should be larger than X. So we don't need to modify the replica version * because its already looks like normal. * Case 3: - * There are new load jobs after alter task, and their version and LFV is smaller or equal to X. + * There are new load jobs after alter task, and their version and LFV is smaller or equal to X. * And because alter request report success, it means that we can increase replica's version to X. */ public void handleFinishAlterTask() throws Exception { diff --git a/fe/fe-core/src/main/java/com/starrocks/task/PushTask.java b/fe/fe-core/src/main/java/com/starrocks/task/PushTask.java index c4cc7d840ce7e..6703487cbb60b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/task/PushTask.java +++ b/fe/fe-core/src/main/java/com/starrocks/task/PushTask.java @@ -45,6 +45,7 @@ import com.starrocks.common.MarkedCountDownLatch; import com.starrocks.common.Status; import com.starrocks.thrift.TBrokerScanRange; +import com.starrocks.thrift.TColumn; import com.starrocks.thrift.TCondition; import com.starrocks.thrift.TDescriptorTable; import com.starrocks.thrift.TPriority; @@ -88,11 +89,14 @@ public class PushTask extends AgentTask { private TTabletType tabletType; + // for light schema change + private List columnsDesc = null; + public PushTask(TResourceInfo resourceInfo, long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, long replicaId, int schemaHash, long version, int timeoutSecond, long loadJobId, TPushType pushType, List conditions, TPriority priority, TTaskType taskType, - long transactionId, long signature) { + long transactionId, long signature, List columnsDesc) { super(resourceInfo, backendId, taskType, dbId, tableId, partitionId, indexId, tabletId, signature); this.replicaId = replicaId; this.schemaHash = schemaHash; @@ -109,6 +113,7 @@ public PushTask(TResourceInfo resourceInfo, long backendId, long dbId, long tabl this.tBrokerScanRange = null; this.tDescriptorTable = null; this.useVectorized = true; + this.columnsDesc = columnsDesc; } // for cancel delete @@ -125,10 +130,10 @@ public PushTask(long backendId, TPushType pushType, TPriority priority, TTaskTyp public PushTask(long backendId, long dbId, long tableId, long partitionId, long indexId, long tabletId, long replicaId, int schemaHash, int timeoutSecond, long loadJobId, TPushType pushType, TPriority priority, long transactionId, long signature, TBrokerScanRange tBrokerScanRange, - TDescriptorTable tDescriptorTable, String timezone, TTabletType tabletType) { + TDescriptorTable tDescriptorTable, String timezone, TTabletType tabletType, List columnsDesc) { this(null, backendId, dbId, tableId, partitionId, indexId, tabletId, replicaId, schemaHash, -1, timeoutSecond, loadJobId, pushType, null, - priority, TTaskType.REALTIME_PUSH, transactionId, signature); + priority, TTaskType.REALTIME_PUSH, transactionId, signature, columnsDesc); this.tBrokerScanRange = tBrokerScanRange; this.tDescriptorTable = tDescriptorTable; this.useVectorized = true; @@ -204,6 +209,7 @@ public TPushReq toThrift() { LOG.warn("unknown push type. type: " + pushType.name()); break; } + request.setColumns_desc(columnsDesc); return request; } diff --git a/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeHandlerTest.java b/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeHandlerTest.java index 66b369bbb742b..d04f34dc38333 100644 --- a/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeHandlerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/alter/SchemaChangeHandlerTest.java @@ -34,45 +34,382 @@ package com.starrocks.alter; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import com.starrocks.analysis.ColumnPosition; import com.starrocks.catalog.Column; -import com.starrocks.catalog.KeysType; +import com.starrocks.catalog.Database; +import com.starrocks.catalog.Index; +import com.starrocks.catalog.MaterializedIndexMeta; import com.starrocks.catalog.OlapTable; -import com.starrocks.common.jmockit.Deencapsulation; -import mockit.Expectations; -import mockit.Injectable; +import com.starrocks.catalog.OlapTable.OlapTableState; +import com.starrocks.catalog.Type; +import com.starrocks.common.DdlException; +import com.starrocks.server.GlobalStateMgr; +import com.starrocks.sql.ast.AlterTableStmt; +import com.starrocks.utframe.TestWithFeService; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; -public class SchemaChangeHandlerTest { +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; - @Test - public void testAddValueColumnOnAggMV(@Injectable OlapTable olapTable, @Injectable Column newColumn, - @Injectable ColumnPosition columnPosition) { - SchemaChangeHandler schemaChangeHandler = new SchemaChangeHandler(); - new Expectations() { - { - olapTable.getKeysType(); - result = KeysType.DUP_KEYS; - newColumn.getAggregationType(); - result = null; - olapTable.getIndexMetaByIndexId(2).getKeysType(); - result = KeysType.AGG_KEYS; - newColumn.isKey(); - result = false; +public class SchemaChangeHandlerTest extends TestWithFeService { + + private static final Logger LOG = LogManager.getLogger(SchemaChangeHandlerTest.class); + private int jobSize = 0; + + @Override + protected void runBeforeAll() throws Exception { + //create database db1 + createDatabase("test"); + + //create tables + String createAggTblStmtStr = "CREATE TABLE IF NOT EXISTS test.sc_agg (\n" + "user_id LARGEINT NOT NULL,\n" + + "date DATE NOT NULL,\n" + "city VARCHAR(20),\n" + "age SMALLINT,\n" + "sex TINYINT,\n" + + "last_visit_date DATETIME REPLACE DEFAULT '1970-01-01 00:00:00',\n" + "cost BIGINT SUM DEFAULT '0',\n" + + "max_dwell_time INT MAX DEFAULT '0',\n" + "min_dwell_time INT MIN DEFAULT '99999')\n" + + "AGGREGATE KEY(user_id, date, city, age, sex)\n" + "DISTRIBUTED BY HASH(user_id) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1', 'light_schema_change' = 'true');"; + createTable(createAggTblStmtStr); + + String createUniqTblStmtStr = "CREATE TABLE IF NOT EXISTS test.sc_uniq (\n" + "user_id LARGEINT NOT NULL,\n" + + "username VARCHAR(50) NOT NULL,\n" + "city VARCHAR(20),\n" + "age SMALLINT,\n" + "sex TINYINT,\n" + + "phone LARGEINT,\n" + "address VARCHAR(500),\n" + "register_time DATETIME)\n" + + "UNIQUE KEY(user_id, username)\n" + "DISTRIBUTED BY HASH(user_id) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1', 'light_schema_change' = 'true');"; + createTable(createUniqTblStmtStr); + + String createDupTblStmtStr = "CREATE TABLE IF NOT EXISTS test.sc_dup (\n" + "timestamp DATETIME,\n" + + "type INT,\n" + "error_code INT,\n" + "error_msg VARCHAR(1024),\n" + "op_id BIGINT,\n" + + "op_time DATETIME)\n" + "DUPLICATE KEY(timestamp, type)\n" + "DISTRIBUTED BY HASH(type) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1', 'light_schema_change' = 'true');"; + + createTable(createDupTblStmtStr); + + String createDupTbl2StmtStr = "CREATE TABLE IF NOT EXISTS test.sc_dup2 (\n" + "timestamp DATETIME,\n" + + "type INT,\n" + "error_code INT,\n" + "error_msg VARCHAR(1024),\n" + "op_id BIGINT,\n" + + "op_time DATETIME)\n" + "DUPLICATE KEY(timestamp, type)\n" + "DISTRIBUTED BY HASH(type) BUCKETS 1\n" + + "PROPERTIES ('replication_num' = '1', 'light_schema_change' = 'true');"; + + createTable(createDupTbl2StmtStr); + + } + + private void waitAlterJobDone(Map alterJobs) throws Exception { + for (AlterJobV2 alterJobV2 : alterJobs.values()) { + while (!alterJobV2.getJobState().isFinalState()) { + LOG.info("alter job {} is running. state: {}", alterJobV2.getJobId(), alterJobV2.getJobState()); + Thread.sleep(1000); + } + LOG.info("alter job {} is done. state: {}", alterJobV2.getJobId(), alterJobV2.getJobState()); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, alterJobV2.getJobState()); + + Database db = GlobalStateMgr.getCurrentState().getDb(alterJobV2.getDbId()); + OlapTable tbl = (OlapTable) db.getTable(alterJobV2.getTableId()); + while (tbl.getState() != OlapTable.OlapTableState.NORMAL) { + Thread.sleep(1000); } - }; + } + } + + @Test + public void testAggAddOrDropColumn() throws Exception { + LOG.info("dbName: {}", GlobalStateMgr.getCurrentState().getDbNames()); + + Database db = GlobalStateMgr.getCurrentState().getDb("test"); + OlapTable tbl = (OlapTable) db.getTable("sc_agg"); + db.readLock(); + try { + Assertions.assertNotNull(tbl); + System.out.println(tbl.getName()); + Assertions.assertEquals("StarRocks", tbl.getEngine()); + Assertions.assertEquals(9, tbl.getBaseSchema().size()); + } finally { + db.readUnlock(); + } + //process agg add value column schema change + String addValColStmtStr = "alter table test.sc_agg add column new_v1 int MAX default '0'"; + AlterTableStmt addValColStmt = (AlterTableStmt) parseAndAnalyzeStmt(addValColStmtStr); + GlobalStateMgr.getCurrentState().getAlterJobMgr().processAlterTable(addValColStmt); + jobSize++; + // check alter job, do not create job + Map alterJobs = GlobalStateMgr.getCurrentState().getSchemaChangeHandler().getAlterJobsV2(); + Assertions.assertEquals(jobSize, alterJobs.size()); + + db.readLock(); + try { + Assertions.assertEquals(10, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + db.readUnlock(); + } + + //process agg add key column schema change + String addKeyColStmtStr = "alter table test.sc_agg add column new_k1 int default '1'"; + AlterTableStmt addKeyColStmt = (AlterTableStmt) parseAndAnalyzeStmt(addKeyColStmtStr); + GlobalStateMgr.getCurrentState().getAlterJobMgr().processAlterTable(addKeyColStmt); + + //check alter job + jobSize++; + Assertions.assertEquals(jobSize, alterJobs.size()); + waitAlterJobDone(alterJobs); + + db.readLock(); try { - Deencapsulation.invoke(schemaChangeHandler, "addColumnInternal", olapTable, newColumn, columnPosition, - new Long(2), new Long(1), - Maps.newHashMap(), Sets.newHashSet()); - Assert.fail(); - } catch (Exception e) { - System.out.println(e.getMessage()); + Assertions.assertEquals(11, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + db.readUnlock(); } + //process agg drop value column schema change + String dropValColStmtStr = "alter table test.sc_agg drop column new_v1"; + AlterTableStmt dropValColStmt = (AlterTableStmt) parseAndAnalyzeStmt(dropValColStmtStr); + GlobalStateMgr.getCurrentState().getAlterJobMgr().processAlterTable(dropValColStmt); + jobSize++; + //check alter job, do not create job + LOG.info("alterJobs:{}", alterJobs); + Assertions.assertEquals(jobSize, alterJobs.size()); + + db.readLock(); + try { + Assertions.assertEquals(10, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + db.readUnlock(); + } + + //process agg drop key column with replace schema change, expect exception. + String dropKeyColStmtStr = "alter table test.sc_agg drop column new_k1"; + AlterTableStmt dropKeyColStmt = (AlterTableStmt) parseAndAnalyzeStmt(dropKeyColStmtStr); + Assertions.assertThrows(Exception.class, + () -> GlobalStateMgr.getCurrentState().getAlterJobMgr().processAlterTable(dropKeyColStmt)); + + LOG.info("getIndexIdToSchema 1: {}", tbl.getIndexIdToSchema()); + + String addRollUpStmtStr = "alter table test.sc_agg add rollup agg_rollup(user_id, max_dwell_time);"; + AlterTableStmt addRollUpStmt = (AlterTableStmt) parseAndAnalyzeStmt(addRollUpStmtStr); + GlobalStateMgr.getCurrentState().getAlterJobMgr().processAlterTable(addRollUpStmt); + // 2. check alter job + Map materializedViewAlterJobs = GlobalStateMgr.getCurrentState().getRollupHandler() + .getAlterJobsV2(); + waitAlterJobDone(materializedViewAlterJobs); + Assertions.assertEquals(1, materializedViewAlterJobs.size()); + + LOG.info("getIndexIdToSchema 2: {}", tbl.getIndexIdToSchema()); + + //process agg drop value column with rollup schema change + String dropRollUpValColStmtStr = "alter table test.sc_agg drop column max_dwell_time"; + AlterTableStmt dropRollUpValColStmt = (AlterTableStmt) parseAndAnalyzeStmt(dropRollUpValColStmtStr); + GlobalStateMgr.getCurrentState().getAlterJobMgr().processAlterTable(dropRollUpValColStmt); + jobSize++; + //check alter job, need create job + LOG.info("alterJobs:{}", alterJobs); + Assertions.assertEquals(jobSize, alterJobs.size()); + waitAlterJobDone(materializedViewAlterJobs); + + db.readLock(); + try { + Assertions.assertEquals(9, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + db.readUnlock(); + } + + //process agg add mul value column schema change + String addMultiValColStmtStr + = "alter table test.sc_agg add column new_v2 int MAX default '0', add column new_v3 int MAX default '1';"; + AlterTableStmt addMultiValColStmt = (AlterTableStmt) parseAndAnalyzeStmt(addMultiValColStmtStr); + GlobalStateMgr.getCurrentState().getAlterJobMgr().processAlterTable(addMultiValColStmt); + jobSize++; + //check alter job, do not create job + Assertions.assertEquals(jobSize, alterJobs.size()); + } + + @Test + public void testUniqAddOrDropColumn() throws Exception { + + LOG.info("dbName: {}", GlobalStateMgr.getCurrentState().getDbNames()); + + Database db = GlobalStateMgr.getCurrentState().getDb("test"); + OlapTable tbl = (OlapTable) db.getTable("sc_uniq"); + db.readLock(); + try { + Assertions.assertNotNull(tbl); + System.out.println(tbl.getName()); + Assertions.assertEquals("StarRocks", tbl.getEngine()); + Assertions.assertEquals(8, tbl.getBaseSchema().size()); + } finally { + db.readUnlock(); + } + + //process uniq add value column schema change + String addValColStmtStr = "alter table test.sc_uniq add column new_v1 int default '0'"; + AlterTableStmt addValColStmt = (AlterTableStmt) parseAndAnalyzeStmt(addValColStmtStr); + GlobalStateMgr.getCurrentState().getAlterJobMgr().processAlterTable(addValColStmt); + jobSize++; + //check alter job, do not create job + Map alterJobs = GlobalStateMgr.getCurrentState().getSchemaChangeHandler().getAlterJobsV2(); + LOG.info("alterJobs:{}", alterJobs); + Assertions.assertEquals(jobSize, alterJobs.size()); + + db.readLock(); + try { + Assertions.assertEquals(9, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + db.readUnlock(); + } + + //process uniq drop val column schema change + String dropValColStmtStr = "alter table test.sc_uniq drop column new_v1"; + AlterTableStmt dropValColStm = (AlterTableStmt) parseAndAnalyzeStmt(dropValColStmtStr); + GlobalStateMgr.getCurrentState().getAlterJobMgr().processAlterTable(dropValColStm); + jobSize++; + //check alter job + Assertions.assertEquals(jobSize, alterJobs.size()); + db.readLock(); + try { + Assertions.assertEquals(8, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + db.readUnlock(); + } + } + + @Test + public void testDupAddOrDropColumn() throws Exception { + + LOG.info("dbName: {}", GlobalStateMgr.getCurrentState().getDbNames()); + + Database db = GlobalStateMgr.getCurrentState().getDb("test"); + OlapTable tbl = (OlapTable) db.getTable("sc_dup"); + db.readLock(); + try { + Assertions.assertNotNull(tbl); + System.out.println(tbl.getName()); + Assertions.assertEquals("StarRocks", tbl.getEngine()); + Assertions.assertEquals(6, tbl.getBaseSchema().size()); + } finally { + db.readUnlock(); + } + + //process uniq add value column schema change + String addValColStmtStr = "alter table test.sc_dup add column new_v1 int default '0'"; + AlterTableStmt addValColStmt = (AlterTableStmt) parseAndAnalyzeStmt(addValColStmtStr); + GlobalStateMgr.getCurrentState().getAlterJobMgr().processAlterTable(addValColStmt); + jobSize++; + //check alter job, do not create job + Map alterJobs = GlobalStateMgr.getCurrentState().getSchemaChangeHandler().getAlterJobsV2(); + LOG.info("alterJobs:{}", alterJobs); + Assertions.assertEquals(jobSize, alterJobs.size()); + + db.readLock(); + try { + Assertions.assertEquals(7, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + db.readUnlock(); + } + + //process uniq drop val column schema change + String dropValColStmtStr = "alter table test.sc_dup drop column new_v1"; + AlterTableStmt dropValColStm = (AlterTableStmt) parseAndAnalyzeStmt(dropValColStmtStr); + GlobalStateMgr.getCurrentState().getAlterJobMgr().processAlterTable(dropValColStm); + jobSize++; + //check alter job + Assertions.assertEquals(jobSize, alterJobs.size()); + db.readLock(); + try { + Assertions.assertEquals(6, tbl.getBaseSchema().size()); + String baseIndexName = tbl.getIndexNameById(tbl.getBaseIndexId()); + Assertions.assertEquals(baseIndexName, tbl.getName()); + MaterializedIndexMeta indexMeta = tbl.getIndexMetaByIndexId(tbl.getBaseIndexId()); + Assertions.assertNotNull(indexMeta); + } finally { + db.readUnlock(); + } + } + + @Test + public void testModifyTableAddOrDropColumns() { + Database db = GlobalStateMgr.getCurrentState().getDb("test"); + OlapTable tbl = (OlapTable) db.getTable("sc_dup2"); + Map alterJobs = GlobalStateMgr.getCurrentState().getSchemaChangeHandler().getAlterJobsV2(); + + // origin columns + Map> indexSchemaMap = new HashMap<>(); + for (Map.Entry> entry : tbl.getIndexIdToSchema().entrySet()) { + indexSchemaMap.put(entry.getKey(), new LinkedList<>(entry.getValue())); + } + List newIndexes = tbl.getCopiedIndexes(); + + Assertions.assertDoesNotThrow( + () -> ((SchemaChangeHandler) GlobalStateMgr.getCurrentState().getAlterJobMgr().getSchemaChangeHandler()) + .modifyTableAddOrDropColumns(db, tbl, indexSchemaMap, newIndexes, 100, false)); + jobSize++; + Assertions.assertEquals(jobSize, alterJobs.size()); + + Assertions.assertDoesNotThrow( + () -> ((SchemaChangeHandler) GlobalStateMgr.getCurrentState().getAlterJobMgr().getSchemaChangeHandler()) + .modifyTableAddOrDropColumns(db, tbl, indexSchemaMap, newIndexes, 101, true)); + jobSize++; + Assertions.assertEquals(jobSize, alterJobs.size()); + + OlapTableState beforeState = tbl.getState(); + tbl.setState(OlapTableState.ROLLUP); + Assertions.assertThrows(DdlException.class, + () -> ((SchemaChangeHandler) GlobalStateMgr.getCurrentState().getAlterJobMgr().getSchemaChangeHandler()) + .modifyTableAddOrDropColumns(db, tbl, indexSchemaMap, newIndexes, 102, false)); + tbl.setState(beforeState); + + Map> indexSchemaMapInvalid2 = new HashMap<>(indexSchemaMap); + + // value before key + indexSchemaMapInvalid2.get(tbl.getBaseIndexId()).add(0, new Column("kk", Type.INT)); + + Assertions.assertThrows(DdlException.class, + () -> ((SchemaChangeHandler) GlobalStateMgr.getCurrentState().getAlterJobMgr().getSchemaChangeHandler()) + .modifyTableAddOrDropColumns(db, tbl, indexSchemaMapInvalid2, newIndexes, 103, false)); + + Map> indexSchemaMapInvalid3 = new HashMap<>(indexSchemaMap); + + // not key + indexSchemaMapInvalid3.get(tbl.getBaseIndexId()).removeIf(Column::isKey); + Assertions.assertThrows(DdlException.class, + () -> ((SchemaChangeHandler) GlobalStateMgr.getCurrentState().getAlterJobMgr().getSchemaChangeHandler()) + .modifyTableAddOrDropColumns(db, tbl, indexSchemaMapInvalid3, newIndexes, 104, false)); + + Map> emptyIndexMap = new HashMap<>(); + + Assertions.assertThrows(DdlException.class, + () -> ((SchemaChangeHandler) GlobalStateMgr.getCurrentState().getAlterJobMgr().getSchemaChangeHandler()) + .modifyTableAddOrDropColumns(db, tbl, emptyIndexMap, newIndexes, 105, false)); + } } diff --git a/fe/fe-core/src/test/java/com/starrocks/catalog/ColumnTest.java b/fe/fe-core/src/test/java/com/starrocks/catalog/ColumnTest.java index bc3f801b47bf8..8f588a50ee806 100644 --- a/fe/fe-core/src/test/java/com/starrocks/catalog/ColumnTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/catalog/ColumnTest.java @@ -34,12 +34,16 @@ package com.starrocks.catalog; +import com.starrocks.analysis.IndexDef.IndexType; +import com.starrocks.analysis.NullLiteral; import com.starrocks.analysis.StringLiteral; import com.starrocks.common.DdlException; import com.starrocks.common.FeConstants; import com.starrocks.common.jmockit.Deencapsulation; import com.starrocks.server.GlobalStateMgr; import com.starrocks.sql.ast.ColumnDef; +import com.starrocks.sql.ast.ColumnDef.DefaultValueDef; +import com.starrocks.thrift.TColumn; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -49,6 +53,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.util.Collections; import static com.starrocks.sql.ast.ColumnDef.DefaultValueDef.CURRENT_TIMESTAMP_VALUE; import static com.starrocks.sql.ast.ColumnDef.DefaultValueDef.NOT_SET; @@ -323,4 +328,24 @@ public void testSchemaChangeAllowedInvolvingDecimalv3() throws DdlException { } } + + @Test + public void testLscColumn() { + Column f0 = new Column("f0", Type.INT, true, AggregateType.NONE, false, + new DefaultValueDef(true, NullLiteral.create(Type.INT)), "", 0); + + Index i0 = new Index("i0", + Collections.singletonList("f0"), IndexType.BITMAP, ""); + + TColumn t0 = f0.toThrift(); + f0.setIndexFlag(t0, Collections.singletonList(i0)); + + Assert.assertEquals(t0.has_bitmap_index, true); + + Assert.assertEquals(f0.getUniqueId(), 0); + f0.setUniqueId(1); + + Assert.assertEquals(f0.getUniqueId(), 1); + + } } diff --git a/fe/fe-core/src/test/java/com/starrocks/common/PropertyAnalyzerTest.java b/fe/fe-core/src/test/java/com/starrocks/common/PropertyAnalyzerTest.java index f83c7dd451027..94f727f78baa5 100644 --- a/fe/fe-core/src/test/java/com/starrocks/common/PropertyAnalyzerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/common/PropertyAnalyzerTest.java @@ -233,4 +233,11 @@ public void testDefaultTableCompression() throws AnalysisException { property.put(PropertyAnalyzer.PROPERTIES_COMPRESSION, "zlib"); Assert.assertEquals(TCompressionType.ZLIB, (PropertyAnalyzer.analyzeCompressionType(property))); } + + @Test + public void testSchemaChangeProperties() throws AnalysisException { + Map props = new HashMap<>(); + props.put(PropertyAnalyzer.PROPERTIES_USE_LIGHT_SCHEMA_CHANGE, "true"); + Assert.assertEquals(PropertyAnalyzer.analyzeUseLightSchemaChange(props), true); + } } diff --git a/fe/fe-core/src/test/java/com/starrocks/persist/TableAddOrDropColumnsInfoTest.java b/fe/fe-core/src/test/java/com/starrocks/persist/TableAddOrDropColumnsInfoTest.java new file mode 100644 index 0000000000000..6c040376fabbb --- /dev/null +++ b/fe/fe-core/src/test/java/com/starrocks/persist/TableAddOrDropColumnsInfoTest.java @@ -0,0 +1,43 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.persist; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Collections; + +public class TableAddOrDropColumnsInfoTest { + @Test + public void test() { + TableAddOrDropColumnsInfo info = new TableAddOrDropColumnsInfo(1, 1, + Collections.emptyMap(), Collections.emptyList(), 0); + + Assert.assertEquals(1, info.getDbId()); + Assert.assertEquals(1, info.getTableId()); + Assert.assertEquals(0, info.getIndexes().size()); + Assert.assertEquals(0, info.getIndexSchemaMap().size()); + Assert.assertEquals(0, info.getJobId()); + + TableAddOrDropColumnsInfo info2 = new TableAddOrDropColumnsInfo(1, 1, + Collections.emptyMap(), Collections.emptyList(), 0); + + Assert.assertEquals(info.hashCode(), info2.hashCode()); + Assert.assertEquals(info, info2); + Assert.assertNotNull(info.toString()); + + } + +} diff --git a/fe/fe-core/src/test/java/com/starrocks/task/AlterReplicaTaskTest.java b/fe/fe-core/src/test/java/com/starrocks/task/AlterReplicaTaskTest.java index 3f6702cbbba31..dd1e9b4d42b29 100644 --- a/fe/fe-core/src/test/java/com/starrocks/task/AlterReplicaTaskTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/task/AlterReplicaTaskTest.java @@ -21,6 +21,7 @@ import org.junit.Assert; import org.junit.Test; +import java.util.Collections; import java.util.HashMap; public class AlterReplicaTaskTest { @@ -28,7 +29,7 @@ public class AlterReplicaTaskTest { @Test public void testAlterLocalTablet() { AlterReplicaTask task = AlterReplicaTask.alterLocalTablet(1, 2, 3, 4, 5, 6, - 7, 8, 9, 10, 11, 12, null); + 7, 8, 9, 10, 11, 12, null, Collections.emptyList()); Assert.assertEquals(1, task.getBackendId()); Assert.assertEquals(2, task.getDbId()); diff --git a/fe/fe-core/src/test/java/com/starrocks/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/com/starrocks/utframe/TestWithFeService.java new file mode 100644 index 0000000000000..4e4e9fad72c87 --- /dev/null +++ b/fe/fe-core/src/test/java/com/starrocks/utframe/TestWithFeService.java @@ -0,0 +1,228 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.starrocks.utframe; + +import com.google.common.collect.ImmutableMap; +import com.starrocks.catalog.DiskInfo; +import com.starrocks.catalog.Replica; +import com.starrocks.catalog.TabletMeta; +import com.starrocks.cluster.ClusterNamespace; +import com.starrocks.common.Config; +import com.starrocks.qe.ConnectContext; +import com.starrocks.qe.OriginStatement; +import com.starrocks.server.GlobalStateMgr; +import com.starrocks.sql.analyzer.Analyzer; +import com.starrocks.sql.ast.CreateTableStmt; +import com.starrocks.sql.ast.DropTableStmt; +import com.starrocks.sql.ast.StatementBase; +import com.starrocks.sql.ast.UserIdentity; +import com.starrocks.sql.parser.SqlParser; +import com.starrocks.system.Backend; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInstance; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Comparator; +import java.util.ConcurrentModificationException; +import java.util.List; +import java.util.UUID; + + +/** + * This is the base class for unit class that wants to start a FE service. + * Concrete test class must be derived class of {@link TestWithFeService}. + * + * This class use {@link TestInstance} in JUnit5 to do initialization and cleanup stuff. Unlike + * deprecated legacy combination-based implementation {@link UtFrameUtils}, we use an + * inherit-manner, + * thus we could wrap common logic in this base class. It's more easy to use. + * Note: + * Unit-test method in derived classes must use the JUnit5 {@link org.junit.jupiter.api.Test} + * annotation, rather than the old JUnit4 {@link org.junit.Test} or others. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class TestWithFeService { + + protected String starrocksHome; + protected String runningDir = + "fe/mocked/" + getClass().getSimpleName() + "/" + UUID.randomUUID() + "/"; + protected ConnectContext connectContext; + + @BeforeAll + public final void beforeAll() throws Exception { + beforeCreatingConnectContext(); + connectContext = createDefaultCtx(); + beforeCluster(); + createStarrocksCluster(); + runBeforeAll(); + } + + protected void beforeCluster() { + } + + @AfterAll + public final void afterAll() throws Exception { + runAfterAll(); + GlobalStateMgr.getCurrentState().clear(); + cleanStarrocksFeDir(); + } + + @BeforeEach + public final void beforeEach() throws Exception { + runBeforeEach(); + } + + protected void beforeCreatingConnectContext() throws Exception { + + } + + protected void runBeforeAll() throws Exception { + } + + protected void runAfterAll() throws Exception { + } + + protected void runBeforeEach() throws Exception { + } + + // Help to create a mocked ConnectContext. + protected ConnectContext createDefaultCtx() throws IOException { + return createCtx(UserIdentity.ROOT, "127.0.0.1"); + } + + protected T createStmt(String showSql) + throws Exception { + return (T) parseAndAnalyzeStmt(showSql, connectContext); + } + + protected ConnectContext createCtx(UserIdentity user, String host) throws IOException { + ConnectContext ctx = new ConnectContext(); + ctx.setCurrentUserIdentity(user); + ctx.setQualifiedUser(user.getQualifiedUser()); + ctx.setRemoteIP(host); + ctx.setGlobalStateMgr(GlobalStateMgr.getCurrentState()); + ctx.setThreadLocalInfo(); + return ctx; + } + + // Parse an origin stmt and analyze it. Return a StatementBase instance. + protected StatementBase parseAndAnalyzeStmt(String originStmt) throws Exception { + return parseAndAnalyzeStmt(originStmt, connectContext); + } + + // Parse an origin stmt and analyze it. Return a StatementBase instance. + protected StatementBase parseAndAnalyzeStmt(String originStmt, ConnectContext ctx) + throws Exception { + System.out.println("begin to parse stmt: " + originStmt); + List statementBases = SqlParser.parse(originStmt, ctx.getSessionVariable()); + StatementBase firstStatement = statementBases.get(0); + Analyzer.analyze(firstStatement, ctx); + firstStatement.setOrigStmt(new OriginStatement(originStmt, 0)); + return firstStatement; + } + + protected void createStarrocksCluster() { + UtFrameUtils.createMinStarRocksCluster(true); + } + + protected void cleanStarrocksFeDir() { + try { + cleanDir(starrocksHome + "/" + runningDir); + cleanDir(Config.plugin_dir); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void createDatabase(String db) throws Exception { + GlobalStateMgr.getCurrentState().getMetadata().createDb(db); + } + + public void useDatabase(String dbName) { + connectContext.setDatabase( + ClusterNamespace.getFullName(dbName)); + } + + public void createTable(String sql) throws Exception { + try { + createTables(sql); + } catch (ConcurrentModificationException e) { + e.printStackTrace(); + throw e; + } + } + + public void dropTable(String table, boolean force) throws Exception { + DropTableStmt dropTableStmt = (DropTableStmt) parseAndAnalyzeStmt( + "drop table " + table + (force ? " force" : "") + ";", connectContext); + GlobalStateMgr.getCurrentState().dropTable(dropTableStmt); + } + + public void createTables(String... sqls) throws Exception { + for (String sql : sqls) { + CreateTableStmt stmt = (CreateTableStmt) parseAndAnalyzeStmt(sql); + GlobalStateMgr.getCurrentState().createTable(stmt); + } + updateReplicaPathHash(); + } + + private void updateReplicaPathHash() { + com.google.common.collect.Table replicaMetaTable = + GlobalStateMgr.getCurrentInvertedIndex() + .getReplicaMetaTable(); + for (com.google.common.collect.Table.Cell cell : replicaMetaTable.cellSet()) { + long beId = cell.getColumnKey(); + Backend be = GlobalStateMgr.getCurrentSystemInfo().getBackend(beId); + if (be == null) { + continue; + } + Replica replica = cell.getValue(); + TabletMeta tabletMeta = + GlobalStateMgr.getCurrentInvertedIndex().getTabletMeta(cell.getRowKey()); + ImmutableMap diskMap = be.getDisks(); + for (DiskInfo diskInfo : diskMap.values()) { + if (diskInfo.getStorageMedium() == tabletMeta.getStorageMedium()) { + replica.setPathHash(diskInfo.getPathHash()); + break; + } + } + } + } + + // clear the specified dir + private void cleanDir(String dir) throws IOException { + File localDir = new File(dir); + if (localDir.exists()) { + Files.walk(Paths.get(dir)) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(file -> { + System.out.println("DELETE FE SERVER DIR: " + file.getAbsolutePath()); + file.delete(); + }); + } else { + System.out.println("No need clean DIR: " + dir); + } + } +} diff --git a/gensrc/proto/descriptors.proto b/gensrc/proto/descriptors.proto index 102a49e60fd38..ee6565f8f10b3 100644 --- a/gensrc/proto/descriptors.proto +++ b/gensrc/proto/descriptors.proto @@ -38,6 +38,7 @@ package starrocks; option java_package = "com.starrocks.proto"; import "types.proto"; +import "tablet_schema.proto"; message PSlotDescriptor { required int32 id = 1; @@ -66,6 +67,7 @@ message POlapTableIndexSchema { required int64 id = 1; repeated string columns = 2; required int32 schema_hash = 3; + repeated ColumnPB columns_desc = 4; }; message POlapTableSchemaParam { diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto index e3ce73b669d23..c6b98e66c1752 100644 --- a/gensrc/proto/olap_file.proto +++ b/gensrc/proto/olap_file.proto @@ -42,7 +42,6 @@ import "binlog.proto"; import "olap_common.proto"; import "tablet_schema.proto"; import "types.proto"; -import "segment.proto"; message ZoneMapOfRowset { required bytes min = 1; @@ -145,6 +144,8 @@ message RowsetMetaPB { optional int64 num_segments = 22; // rowset id definition, it will replace required rowset id optional string rowset_id = 23; + // tablet meta pb, for compaction + optional TabletSchemaPB tablet_schema = 24; // to indicate whether the data between the segments overlap optional SegmentsOverlapPB segments_overlap_pb = 51 [default = OVERLAP_UNKNOWN]; // every segment in a rowset has an unique uint32 id diff --git a/gensrc/proto/tablet_schema.proto b/gensrc/proto/tablet_schema.proto index d680927a9c68a..9d7a3d7120665 100644 --- a/gensrc/proto/tablet_schema.proto +++ b/gensrc/proto/tablet_schema.proto @@ -82,6 +82,7 @@ message TabletSchemaPB { optional int64 deprecated_id = 9; // deprecated optional CompressionTypePB compression_type = 10 [default=LZ4_FRAME]; repeated uint32 sort_key_idxes = 11; + optional int32 schema_version = 12; optional int64 id = 50; } diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index dff59bd563fa0..52a852e809ef0 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -145,6 +145,7 @@ struct TAlterTabletReqV2 { 11: optional i64 job_id 12: optional InternalService.TQueryGlobals query_globals 13: optional InternalService.TQueryOptions query_options + 14: optional list columns } struct TAlterMaterializedViewParam { @@ -178,6 +179,7 @@ struct TPushReq { // 14 and 15 are used by spark load 14: optional PlanNodes.TBrokerScanRange broker_scan_range 15: optional Descriptors.TDescriptorTable desc_tbl + 16: optional list columns_desc 30: optional bool use_vectorized // 31 are used by spark load diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 0a6cd7916d2ec..2d04f073d1218 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -51,6 +51,7 @@ struct TSlotDescriptor { 10: optional bool isMaterialized // Deprecated 11: optional bool isOutputColumn // Deprecated 12: optional bool isNullable // replace nullIndicatorBit & nullIndicatorByte + 13: optional i32 col_unique_id = -1 } struct TTupleDescriptor { @@ -197,7 +198,9 @@ struct TColumn { 6: optional string default_value 7: optional bool is_bloom_filter_column 8: optional Exprs.TExpr define_expr - 9: optional bool is_auto_increment + 9: optional bool is_auto_increment + 10: optional i32 col_unique_id = -1 + 11: optional bool has_bitmap_index = false // How many bytes used for short key index encoding. // For fixed-length column, this value may be ignored by BE when creating a tablet. @@ -256,6 +259,7 @@ struct TOlapTableIndexSchema { 1: required i64 id 2: required list columns 3: required i32 schema_hash + 4: required list columns_desc } struct TOlapTableSchemaParam { diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index a8d44de3dcac9..22876e71d0f05 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -491,7 +491,9 @@ struct TOlapScanNode { 27: optional list sort_key_column_names 28: optional i32 max_parallel_scan_instance_num 29: optional list column_access_paths + 30: optional bool use_pk_index + 31: required list columns_desc } struct TJDBCScanNode { diff --git a/test/sql/test_light_schema_change/R/test_light_schema_change b/test/sql/test_light_schema_change/R/test_light_schema_change new file mode 100644 index 0000000000000..af86fa39750e9 --- /dev/null +++ b/test/sql/test_light_schema_change/R/test_light_schema_change @@ -0,0 +1,85 @@ +-- name: test_light_schema_change +create database test_light_schema_change; +-- result: +-- !result +use test_light_schema_change; +-- result: +-- !result +create table t1(k int, v int not null) ENGINE=OLAP DUPLICATE KEY(k) PROPERTIES ("replication_num" = "1", 'light_schema_change' = 'true'); +-- result: +-- !result +insert into t1 values(1, 1); +-- result: +-- !result +select * from t1 order by k; +-- result: +1 1 +-- !result +alter table t1 add column (v1 int, v2 int, v3 int); +-- result: +-- !result +select * from t1 order by k; +-- result: +1 1 None None None +-- !result +insert into t1 values(2, 2, 3, 4, 5); +-- result: +-- !result +select * from t1 order by k; +-- result: +1 1 None None None +2 2 3 4 5 +-- !result +alter table t1 drop column v; +-- result: +-- !result +alter table t1 drop column v2; +-- result: +-- !result +insert into t1 values(2, 2, 3); +-- result: +-- !result +select * from t1 order by k; +-- result: +1 None None +2 2 3 +2 3 5 +-- !result +alter table t1 add column k2 int key; +-- result: +-- !result +function: wait_alter_table_finish() +-- result: +None +-- !result +select * from t1 order by k; +-- result: +1 None None None +2 None 2 3 +2 None 3 5 +-- !result +insert into t1 values(3, 2, 3, 4); +-- result: +-- !result +select * from t1 order by k; +-- result: +1 None None None +2 None 2 3 +2 None 3 5 +3 2 3 4 +-- !result +delete from t1 where v3>4; +-- result: +-- !result +select * from t1 order by k; +-- result: +1 None None None +2 None 2 3 +3 2 3 4 +-- !result +drop table t1; +-- result: +-- !result +drop database test_light_schema_change; +-- result: +-- !result \ No newline at end of file diff --git a/test/sql/test_light_schema_change/T/test_light_schema_change b/test/sql/test_light_schema_change/T/test_light_schema_change new file mode 100644 index 0000000000000..26f684c9dff20 --- /dev/null +++ b/test/sql/test_light_schema_change/T/test_light_schema_change @@ -0,0 +1,30 @@ +-- name: test_light_schema_change +create database test_light_schema_change; +use test_light_schema_change; +create table t1(k int, v int not null) ENGINE=OLAP DUPLICATE KEY(k) PROPERTIES ("replication_num" = "1", 'light_schema_change' = 'true'); +insert into t1 values(1, 1); +select * from t1 order by k; + +alter table t1 add column (v1 int, v2 int, v3 int); +select * from t1 order by k; +insert into t1 values(2, 2, 3, 4, 5); +select * from t1 order by k; +alter table t1 drop column v; + +alter table t1 drop column v2; +insert into t1 values(2, 2, 3); +select * from t1 order by k; + +alter table t1 add column k2 int key; +function: wait_alter_table_finish() + +select * from t1 order by k; +insert into t1 values(3, 2, 3, 4); +select * from t1 order by k; + + +delete from t1 where v3>4; +select * from t1 order by k; + +drop table t1; +drop database test_light_schema_change; \ No newline at end of file