Skip to content

Commit

Permalink
[Alter] Add more schema change to varchar type (apache#2777)
Browse files Browse the repository at this point in the history
  • Loading branch information
@WingsGo
WingsGo authored Feb 19, 2020
1 parent c617fc9 commit cc0d412
Show file tree
Hide file tree
Showing 5 changed files with 116 additions and 4 deletions.
28 changes: 27 additions & 1 deletion be/src/olap/types.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -477,6 +477,12 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_FLOAT> : public BaseFieldtypeTraits<OLAP_
*reinterpret_cast<CppType*>(buf) = value;
return OLAP_SUCCESS;
}
static std::string to_string(const void* src) {
char buf[1024] = {'\0'};
int length = FloatToBuffer(*reinterpret_cast<const CppType *>(src), MAX_FLOAT_STR_LENGTH, buf);
DCHECK(length >= 0) << "gcvt float failed, float value=" << *reinterpret_cast<const CppType *>(src);
return std::string(buf);
}
static OLAPStatus convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool) {
if (src_type->type() == OLAP_FIELD_TYPE_VARCHAR) {
return convert_float_from_varchar<CppType>(dest, src);
Expand All @@ -497,7 +503,8 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_DOUBLE> : public BaseFieldtypeTraits<OLAP
}
static std::string to_string(const void* src) {
char buf[1024] = {'\0'};
snprintf(buf, sizeof(buf), "%.10f", *reinterpret_cast<const CppType*>(src));
int length = DoubleToBuffer(*reinterpret_cast<const CppType *>(src), MAX_DOUBLE_STR_LENGTH, buf);
DCHECK(length >= 0) << "gcvt float failed, float value=" << *reinterpret_cast<const CppType *>(src);
return std::string(buf);
}
static OLAPStatus convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool) {
Expand Down Expand Up @@ -770,6 +777,25 @@ struct FieldTypeTraits<OLAP_FIELD_TYPE_VARCHAR> : public FieldTypeTraits<OLAP_FI
return OLAP_SUCCESS;
}

static OLAPStatus convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool) {
if (src_type->type() == OLAP_FIELD_TYPE_TINYINT
|| src_type->type() == OLAP_FIELD_TYPE_SMALLINT
|| src_type->type() == OLAP_FIELD_TYPE_INT
|| src_type->type() == OLAP_FIELD_TYPE_BIGINT
|| src_type->type() == OLAP_FIELD_TYPE_LARGEINT
|| src_type->type() == OLAP_FIELD_TYPE_FLOAT
|| src_type->type() == OLAP_FIELD_TYPE_DOUBLE
|| src_type->type() == OLAP_FIELD_TYPE_DECIMAL) {
auto result = src_type->to_string(src);
auto slice = reinterpret_cast<Slice*>(dest);
slice->data = reinterpret_cast<char*>(mem_pool->allocate(result.size()));
memcpy(slice->data, result.c_str(), result.size());
slice->size = result.size();
return OLAP_SUCCESS;
}
return OLAP_ERR_INVALID_SCHEMA;
}

static void set_to_min(void* buf) {
auto slice = reinterpret_cast<Slice*>(buf);
slice->size = 0;
Expand Down
77 changes: 75 additions & 2 deletions be/test/olap/schema_change_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,48 @@ class TestColumn : public testing::Test {
ASSERT_EQ(_column_writer->create_row_index_entry(), OLAP_SUCCESS);
}

void test_convert_from_varchar(std::string type_name, int type_size, const std::string& value, OLAPStatus expected_st) {
template <typename T>
void test_convert_to_varchar(const std::string& type_name, int type_size, T val, const std::string& expected_val, OLAPStatus expected_st) {
TabletSchema src_tablet_schema;
SetTabletSchema("ConvertColumn", type_name, "REPLACE", type_size, false, false, &src_tablet_schema);
CreateColumnWriter(src_tablet_schema);

RowCursor write_row;
write_row.init(src_tablet_schema);
RowBlock block(&src_tablet_schema);
RowBlockInfo block_info;
block_info.row_num = 10000;
block.init(block_info);
write_row.set_field_content(0, reinterpret_cast<char*>(&val), _mem_pool.get());
block.set_row(0, write_row);
block.finalize(1);
ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS);
ColumnDataHeaderMessage header;
ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS);
helper.close();

TabletSchema dst_tablet_schema;
SetTabletSchema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &dst_tablet_schema);
CreateColumnReader(src_tablet_schema);
RowCursor read_row;
read_row.init(dst_tablet_schema);

_col_vector.reset(new ColumnVector());
ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS);
char* data = reinterpret_cast<char*>(_col_vector->col_data());
auto st = read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get());
ASSERT_EQ(st, expected_st);
if (st == OLAP_SUCCESS) {
std::string dst_str = read_row.column_schema(0)->to_string(read_row.cell_ptr(0));
ASSERT_TRUE(dst_str.compare(0, expected_val.size(), expected_val) == 0);
}

TypeInfo* tp = get_type_info(OLAP_FIELD_TYPE_HLL);
st = read_row.convert_from(0, read_row.cell_ptr(0), tp, _mem_pool.get());
ASSERT_EQ(st, OLAP_ERR_INVALID_SCHEMA);
}

void test_convert_from_varchar(const std::string& type_name, int type_size, const std::string& value, OLAPStatus expected_st) {
TabletSchema tablet_schema;
SetTabletSchema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &tablet_schema);
CreateColumnWriter(tablet_schema);
Expand All @@ -230,8 +271,8 @@ class TestColumn : public testing::Test {
ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS);
ColumnDataHeaderMessage header;
ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS);

helper.close();

TabletSchema converted_tablet_schema;
SetTabletSchema("ConvertColumn", type_name, "REPLACE", type_size, false, false, &converted_tablet_schema);
CreateColumnReader(tablet_schema);
Expand Down Expand Up @@ -560,6 +601,38 @@ TEST_F(TestColumn, ConvertVarcharToDouble) {
"1797690000000000063230304921389426434930330364336853362154109832891264341489062899406152996321966094455338163203127744334848599000464911410516510916727344709727599413825823048028128827530592629736371829425359826368844446113768685826367454055532068818593409163400929532301499014067384276511218551077374242324480.0000000000", OLAP_ERR_INVALID_SCHEMA);
}

TEST_F(TestColumn, ConvertTinyIntToVarchar) {
test_convert_to_varchar<int8_t>("TINYINT", 1, 127, "127", OLAP_SUCCESS);
}

TEST_F(TestColumn, ConvertSmallIntToVarchar) {
test_convert_to_varchar<int16_t>("SMALLINT", 2, 32767, "32767", OLAP_SUCCESS);
}

TEST_F(TestColumn, ConvertIntToVarchar) {
test_convert_to_varchar<int32_t>("INT", 4, 2147483647, "2147483647", OLAP_SUCCESS);
}

TEST_F(TestColumn, ConvertBigIntToVarchar) {
test_convert_to_varchar<int64_t>("BIGINT", 8, 9223372036854775807, "9223372036854775807", OLAP_SUCCESS);
}

TEST_F(TestColumn, ConvertLargeIntToVarchar) {
test_convert_to_varchar<int128_t>("LARGEINT", 16, 1701411834604690, "1701411834604690", OLAP_SUCCESS);
}

TEST_F(TestColumn, ConvertFloatToVarchar) {
test_convert_to_varchar<float>("FLOAT", 4, 3.40282e+38, "3.40282e+38", OLAP_SUCCESS);
}

TEST_F(TestColumn, ConvertDoubleToVarchar) {
test_convert_to_varchar<double>("DOUBLE", 8, 123.456, "123.456", OLAP_SUCCESS);
}

TEST_F(TestColumn, ConvertDecimalToVarchar) {
decimal12_t val(456, 789000000);
test_convert_to_varchar<decimal12_t>("Decimal", 12, val, "456.789000000", OLAP_SUCCESS);
}
}

int main(int argc, char** argv) {
Expand Down
1 change: 1 addition & 0 deletions docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ under the License.
4) 分区列不能做任何修改
5) 目前支持以下类型的转换（精度损失由用户保证）
TINYINT/SMALLINT/INT/BIGINT 转换成 TINYINT/SMALLINT/INT/BIGINT/DOUBLE。
TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE/DECIMAL 转换成 VARCHAR
LARGEINT 转换成 DOUBLE
VARCHAR 支持修改最大长度
VARCHAR 转换成 TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE
Expand Down
1 change: 1 addition & 0 deletions ...documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ under the License.
4) The partition column cannot be modified
5) The following types of conversions are currently supported (accuracy loss is guaranteed by the user)
TINYINT/SMALLINT/INT/BIGINT is converted to TINYINT/SMALLINT/INT/BIGINT/DOUBLE.
TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE/DECIMAL is converted to VARCHAR
Convert LARGEINT to DOUBLE
VARCHAR supports modification of maximum length
Convert VARCHAR to TINYINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE.
Expand Down
13 changes: 12 additions & 1 deletion fe/src/main/java/org/apache/doris/catalog/ColumnType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,19 +47,30 @@ public abstract class ColumnType {
schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.BIGINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;

schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.INT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.BIGINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;

schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.BIGINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.DATE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;

schemaChangeMatrix[PrimitiveType.BIGINT.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.BIGINT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.BIGINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;

schemaChangeMatrix[PrimitiveType.LARGEINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;

schemaChangeMatrix[PrimitiveType.FLOAT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.FLOAT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;

schemaChangeMatrix[PrimitiveType.DOUBLE.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;

schemaChangeMatrix[PrimitiveType.CHAR.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.CHAR.ordinal()][PrimitiveType.CHAR.ordinal()] = true;
Expand All @@ -74,8 +85,8 @@ public abstract class ColumnType {
schemaChangeMatrix[PrimitiveType.VARCHAR.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true;

schemaChangeMatrix[PrimitiveType.DATETIME.ordinal()][PrimitiveType.DATE.ordinal()] = true;

schemaChangeMatrix[PrimitiveType.DATE.ordinal()][PrimitiveType.DATETIME.ordinal()] = true;
schemaChangeMatrix[PrimitiveType.FLOAT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true;
}

static boolean isSchemaChangeAllowed(Type lhs, Type rhs) {
Expand Down

0 comments on commit cc0d412

Please sign in to comment.