Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support materialized view schema change [#3739] #3873

Merged
merged 9 commits into from
Jul 16, 2020
Merged

Support materialized view schema change [#3739] #3873

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
60d39c7
add materialized view function
HangyuanLiu Jun 15, 2020
5c927f4
add materialized view function
HangyuanLiu Jun 15, 2020
6696e85
Merge branch 'master' of https://github.com/apache/incubator-doris in…
HangyuanLiu Jun 17, 2020
4fe208f
fix bug in to_bitmap function
HangyuanLiu Jul 8, 2020
db42e5e
fix bug
HangyuanLiu Jul 10, 2020
7ce87b3
Merge branch 'mv_func' of https://github.com/HangyuanLiu/incubator-do…
HangyuanLiu Jul 10, 2020
fe17c1c
fix bug
HangyuanLiu Jul 10, 2020
d06c0ca
Merge remote-tracking branch 'upstream/master' into mv_func
HangyuanLiu Jul 14, 2020
63c6b28
fix compile
HangyuanLiu Jul 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge remote-tracking branch 'upstream/master' into mv_func 
# Conflicts:
#	be/src/olap/schema_change.cpp
#	be/src/olap/schema_change.h
  • Loading branch information
@HangyuanLiu
HangyuanLiu committed Jul 14, 2020
commit d06c0ca4aba12c1be50384406ad7a56d76f9a3f1
137 changes: 128 additions & 9 deletions be/src/olap/schema_change.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@

#include "olap/schema_change.h"

#include <csignal>
#include <pthread.h>
#include <signal.h>

#include <algorithm>
#include <vector>

Expand All @@ -29,10 +31,13 @@
#include "olap/row_cursor.h"
#include "olap/rowset/rowset_factory.h"
#include "olap/rowset/rowset_id_generator.h"
#include "olap/storage_engine.h"
#include "olap/tablet.h"
#include "olap/wrapper_field.h"
#include "runtime/exec_env.h"
#include "runtime/heartbeat_flags.h"
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
#include "agent/cgroups_mgr.h"
#include "runtime/exec_env.h"

using std::deque;
using std::list;
Expand Down Expand Up @@ -86,7 +91,6 @@ class RowBlockMerger {
std::priority_queue<MergeElement> _heap;
};


RowBlockChanger::RowBlockChanger(const TabletSchema& tablet_schema) {
_schema_mapping.resize(tablet_schema.num_columns());
}
Expand Down Expand Up @@ -129,6 +133,124 @@ ColumnMapping* RowBlockChanger::get_mutable_column_mapping(size_t column_index)
break; \
}

#define LARGEINT_REINTERPRET_CAST(FromType, ToType) \
{ \
size_t row_num = ref_block->row_block_info().row_num; \
for (size_t row = 0, mutable_row = 0; row < row_num; ++row) { \
if (is_data_left_vec[row] != 0) { \
char* ref_ptr = ref_block->field_ptr(row, ref_column); \
char* new_ptr = mutable_block->field_ptr(mutable_row++, i); \
*new_ptr = *ref_ptr; \
ToType new_value = *(FromType*)(ref_ptr + 1); \
memcpy(new_ptr + 1, &new_value, sizeof(ToType)); \
} \
} \
break; \
}

#define CONVERT_FROM_TYPE(from_type) \
{ \
switch (mutable_block->tablet_schema().column(i).type()) { \
case OLAP_FIELD_TYPE_TINYINT: \
TYPE_REINTERPRET_CAST(from_type, int8_t); \
case OLAP_FIELD_TYPE_UNSIGNED_TINYINT: \
TYPE_REINTERPRET_CAST(from_type, uint8_t); \
case OLAP_FIELD_TYPE_SMALLINT: \
TYPE_REINTERPRET_CAST(from_type, int16_t); \
case OLAP_FIELD_TYPE_UNSIGNED_SMALLINT: \
TYPE_REINTERPRET_CAST(from_type, uint16_t); \
case OLAP_FIELD_TYPE_INT: \
TYPE_REINTERPRET_CAST(from_type, int32_t); \
case OLAP_FIELD_TYPE_UNSIGNED_INT: \
TYPE_REINTERPRET_CAST(from_type, uint32_t); \
case OLAP_FIELD_TYPE_BIGINT: \
TYPE_REINTERPRET_CAST(from_type, int64_t); \
case OLAP_FIELD_TYPE_UNSIGNED_BIGINT: \
TYPE_REINTERPRET_CAST(from_type, uint64_t); \
case OLAP_FIELD_TYPE_LARGEINT: \
LARGEINT_REINTERPRET_CAST(from_type, int128_t); \
case OLAP_FIELD_TYPE_DOUBLE: \
TYPE_REINTERPRET_CAST(from_type, double); \
default: \
LOG(WARNING) << "the column type which was altered to was unsupported." \
<< " origin_type=" \
<< ref_block->tablet_schema().column(ref_column).type() \
<< ", alter_type=" << mutable_block->tablet_schema().column(i).type(); \
return false; \
} \
break; \
}

#define ASSIGN_DEFAULT_VALUE(length) \
case length: { \
for (size_t row = 0; row < ref_block.row_block_info().row_num; ++row) { \
memcpy(buf, _schema_mapping[i].default_value->ptr(), length); \
buf += length; \
} \
break; \
}

struct ConvertTypeMapHash {
size_t operator()(const std::pair<FieldType, FieldType>& pair) const {
return (pair.first + 31) ^ pair.second;
}
};

class ConvertTypeResolver {
DECLARE_SINGLETON(ConvertTypeResolver);

public:
bool get_convert_type_info(const FieldType from_type, const FieldType to_type) const {
return _convert_type_set.find(std::make_pair(from_type, to_type)) !=
_convert_type_set.end();
}

template <FieldType from_type, FieldType to_type>
void add_convert_type_mapping() {
_convert_type_set.emplace(std::make_pair(from_type, to_type));
}

private:
typedef std::pair<FieldType, FieldType> convert_type_pair;
std::unordered_set<convert_type_pair, ConvertTypeMapHash> _convert_type_set;

DISALLOW_COPY_AND_ASSIGN(ConvertTypeResolver);
};

ConvertTypeResolver::ConvertTypeResolver() {
// supported type convert should annotate in doc:
// http://doris.incubator.apache.org/master/zh-CN/sql-reference/sql-statements/Data%20Definition/ALTER%20TABLE.html#description
// If type convert is supported here, you should check fe/src/main/java/org/apache/doris/catalog/ColumnType.java to supported it either
// from varchar type
add_convert_type_mapping<OLAP_FIELD_TYPE_VARCHAR, OLAP_FIELD_TYPE_TINYINT>();
add_convert_type_mapping<OLAP_FIELD_TYPE_VARCHAR, OLAP_FIELD_TYPE_SMALLINT>();
add_convert_type_mapping<OLAP_FIELD_TYPE_VARCHAR, OLAP_FIELD_TYPE_INT>();
add_convert_type_mapping<OLAP_FIELD_TYPE_VARCHAR, OLAP_FIELD_TYPE_BIGINT>();
add_convert_type_mapping<OLAP_FIELD_TYPE_VARCHAR, OLAP_FIELD_TYPE_LARGEINT>();
add_convert_type_mapping<OLAP_FIELD_TYPE_VARCHAR, OLAP_FIELD_TYPE_FLOAT>();
add_convert_type_mapping<OLAP_FIELD_TYPE_VARCHAR, OLAP_FIELD_TYPE_DOUBLE>();
add_convert_type_mapping<OLAP_FIELD_TYPE_VARCHAR, OLAP_FIELD_TYPE_DATE>();

// to varchar type
add_convert_type_mapping<OLAP_FIELD_TYPE_TINYINT, OLAP_FIELD_TYPE_VARCHAR>();
add_convert_type_mapping<OLAP_FIELD_TYPE_SMALLINT, OLAP_FIELD_TYPE_VARCHAR>();
add_convert_type_mapping<OLAP_FIELD_TYPE_INT, OLAP_FIELD_TYPE_VARCHAR>();
add_convert_type_mapping<OLAP_FIELD_TYPE_BIGINT, OLAP_FIELD_TYPE_VARCHAR>();
add_convert_type_mapping<OLAP_FIELD_TYPE_LARGEINT, OLAP_FIELD_TYPE_VARCHAR>();
add_convert_type_mapping<OLAP_FIELD_TYPE_FLOAT, OLAP_FIELD_TYPE_VARCHAR>();
add_convert_type_mapping<OLAP_FIELD_TYPE_DOUBLE, OLAP_FIELD_TYPE_VARCHAR>();
add_convert_type_mapping<OLAP_FIELD_TYPE_DECIMAL, OLAP_FIELD_TYPE_VARCHAR>();

add_convert_type_mapping<OLAP_FIELD_TYPE_DATE, OLAP_FIELD_TYPE_DATETIME>();

add_convert_type_mapping<OLAP_FIELD_TYPE_DATETIME, OLAP_FIELD_TYPE_DATE>();

add_convert_type_mapping<OLAP_FIELD_TYPE_FLOAT, OLAP_FIELD_TYPE_DOUBLE>();

add_convert_type_mapping<OLAP_FIELD_TYPE_INT, OLAP_FIELD_TYPE_DATE>();
}

ConvertTypeResolver::~ConvertTypeResolver() {}

bool to_bitmap(RowCursor* read_helper, RowCursor* write_helper, const TabletColumn& ref_column,
int field_idx, int ref_field_idx, MemPool* mem_pool) {
Expand Down Expand Up @@ -263,11 +385,8 @@ bool count_field(RowCursor* read_helper, RowCursor* write_helper, const TabletCo
return true;
}

bool RowBlockChanger::change_row_block(
const RowBlock* ref_block,
int32_t data_version,
RowBlock* mutable_block,
uint64_t* filtered_rows) const {
bool RowBlockChanger::change_row_block(const RowBlock* ref_block, int32_t data_version,
RowBlock* mutable_block, uint64_t* filtered_rows) const {
if (mutable_block == nullptr) {
LOG(FATAL) << "mutable block is uninitialized.";
return false;
Expand Down
3 changes: 1 addition & 2 deletions be/src/olap/schema_change.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,7 @@ bool count_field(RowCursor* read_helper, RowCursor* write_helper, const TabletCo

class RowBlockChanger {
public:
RowBlockChanger(const TabletSchema& tablet_schema,
const DeleteHandler& delete_handler);
RowBlockChanger(const TabletSchema& tablet_schema， const DeleteHandler& delete_handler);

RowBlockChanger(const TabletSchema& tablet_schema);

Expand Down
You are viewing a condensed version of this merge commit. You can view the full changes here.