From cc0d41277c8879691e6743663fbfe4def8dd0042 Mon Sep 17 00:00:00 2001 From: WingC <1018957763@qq.com> Date: Wed, 19 Feb 2020 09:14:43 -0600 Subject: [PATCH] [Alter] Add more schema change to varchar type (#2777) --- be/src/olap/types.h | 28 ++++++- be/test/olap/schema_change_test.cpp | 77 ++++++++++++++++++- .../Data Definition/ALTER TABLE.md | 1 + .../Data Definition/ALTER TABLE_EN.md | 1 + .../org/apache/doris/catalog/ColumnType.java | 13 +++- 5 files changed, 116 insertions(+), 4 deletions(-) diff --git a/be/src/olap/types.h b/be/src/olap/types.h index e2436db4219c22..09dd37df785c99 100644 --- a/be/src/olap/types.h +++ b/be/src/olap/types.h @@ -477,6 +477,12 @@ struct FieldTypeTraits : public BaseFieldtypeTraits(buf) = value; return OLAP_SUCCESS; } + static std::string to_string(const void* src) { + char buf[1024] = {'\0'}; + int length = FloatToBuffer(*reinterpret_cast(src), MAX_FLOAT_STR_LENGTH, buf); + DCHECK(length >= 0) << "gcvt float failed, float value=" << *reinterpret_cast(src); + return std::string(buf); + } static OLAPStatus convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool) { if (src_type->type() == OLAP_FIELD_TYPE_VARCHAR) { return convert_float_from_varchar(dest, src); @@ -497,7 +503,8 @@ struct FieldTypeTraits : public BaseFieldtypeTraits(src)); + int length = DoubleToBuffer(*reinterpret_cast(src), MAX_DOUBLE_STR_LENGTH, buf); + DCHECK(length >= 0) << "gcvt float failed, float value=" << *reinterpret_cast(src); return std::string(buf); } static OLAPStatus convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool) { @@ -770,6 +777,25 @@ struct FieldTypeTraits : public FieldTypeTraitstype() == OLAP_FIELD_TYPE_TINYINT + || src_type->type() == OLAP_FIELD_TYPE_SMALLINT + || src_type->type() == OLAP_FIELD_TYPE_INT + || src_type->type() == OLAP_FIELD_TYPE_BIGINT + || src_type->type() == OLAP_FIELD_TYPE_LARGEINT + || src_type->type() == OLAP_FIELD_TYPE_FLOAT + || src_type->type() == OLAP_FIELD_TYPE_DOUBLE + || src_type->type() == OLAP_FIELD_TYPE_DECIMAL) { + auto result = src_type->to_string(src); + auto slice = reinterpret_cast(dest); + slice->data = reinterpret_cast(mem_pool->allocate(result.size())); + memcpy(slice->data, result.c_str(), result.size()); + slice->size = result.size(); + return OLAP_SUCCESS; + } + return OLAP_ERR_INVALID_SCHEMA; + } + static void set_to_min(void* buf) { auto slice = reinterpret_cast(buf); slice->size = 0; diff --git a/be/test/olap/schema_change_test.cpp b/be/test/olap/schema_change_test.cpp index cf9e68a5310cbb..0d2b931ff1e1b1 100644 --- a/be/test/olap/schema_change_test.cpp +++ b/be/test/olap/schema_change_test.cpp @@ -212,7 +212,48 @@ class TestColumn : public testing::Test { ASSERT_EQ(_column_writer->create_row_index_entry(), OLAP_SUCCESS); } - void test_convert_from_varchar(std::string type_name, int type_size, const std::string& value, OLAPStatus expected_st) { + template + void test_convert_to_varchar(const std::string& type_name, int type_size, T val, const std::string& expected_val, OLAPStatus expected_st) { + TabletSchema src_tablet_schema; + SetTabletSchema("ConvertColumn", type_name, "REPLACE", type_size, false, false, &src_tablet_schema); + CreateColumnWriter(src_tablet_schema); + + RowCursor write_row; + write_row.init(src_tablet_schema); + RowBlock block(&src_tablet_schema); + RowBlockInfo block_info; + block_info.row_num = 10000; + block.init(block_info); + write_row.set_field_content(0, reinterpret_cast(&val), _mem_pool.get()); + block.set_row(0, write_row); + block.finalize(1); + ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); + ColumnDataHeaderMessage header; + ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); + helper.close(); + + TabletSchema dst_tablet_schema; + SetTabletSchema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &dst_tablet_schema); + CreateColumnReader(src_tablet_schema); + RowCursor read_row; + read_row.init(dst_tablet_schema); + + _col_vector.reset(new ColumnVector()); + ASSERT_EQ(_column_reader->next_vector(_col_vector.get(), 1, _mem_pool.get()), OLAP_SUCCESS); + char* data = reinterpret_cast(_col_vector->col_data()); + auto st = read_row.convert_from(0, data, write_row.column_schema(0)->type_info(), _mem_pool.get()); + ASSERT_EQ(st, expected_st); + if (st == OLAP_SUCCESS) { + std::string dst_str = read_row.column_schema(0)->to_string(read_row.cell_ptr(0)); + ASSERT_TRUE(dst_str.compare(0, expected_val.size(), expected_val) == 0); + } + + TypeInfo* tp = get_type_info(OLAP_FIELD_TYPE_HLL); + st = read_row.convert_from(0, read_row.cell_ptr(0), tp, _mem_pool.get()); + ASSERT_EQ(st, OLAP_ERR_INVALID_SCHEMA); + } + + void test_convert_from_varchar(const std::string& type_name, int type_size, const std::string& value, OLAPStatus expected_st) { TabletSchema tablet_schema; SetTabletSchema("VarcharColumn", "VARCHAR", "REPLACE", 255, false, false, &tablet_schema); CreateColumnWriter(tablet_schema); @@ -230,8 +271,8 @@ class TestColumn : public testing::Test { ASSERT_EQ(_column_writer->write_batch(&block, &write_row), OLAP_SUCCESS); ColumnDataHeaderMessage header; ASSERT_EQ(_column_writer->finalize(&header), OLAP_SUCCESS); - helper.close(); + TabletSchema converted_tablet_schema; SetTabletSchema("ConvertColumn", type_name, "REPLACE", type_size, false, false, &converted_tablet_schema); CreateColumnReader(tablet_schema); @@ -560,6 +601,38 @@ TEST_F(TestColumn, ConvertVarcharToDouble) { "1797690000000000063230304921389426434930330364336853362154109832891264341489062899406152996321966094455338163203127744334848599000464911410516510916727344709727599413825823048028128827530592629736371829425359826368844446113768685826367454055532068818593409163400929532301499014067384276511218551077374242324480.0000000000", OLAP_ERR_INVALID_SCHEMA); } +TEST_F(TestColumn, ConvertTinyIntToVarchar) { + test_convert_to_varchar("TINYINT", 1, 127, "127", OLAP_SUCCESS); +} + +TEST_F(TestColumn, ConvertSmallIntToVarchar) { + test_convert_to_varchar("SMALLINT", 2, 32767, "32767", OLAP_SUCCESS); +} + +TEST_F(TestColumn, ConvertIntToVarchar) { + test_convert_to_varchar("INT", 4, 2147483647, "2147483647", OLAP_SUCCESS); +} + +TEST_F(TestColumn, ConvertBigIntToVarchar) { + test_convert_to_varchar("BIGINT", 8, 9223372036854775807, "9223372036854775807", OLAP_SUCCESS); +} + +TEST_F(TestColumn, ConvertLargeIntToVarchar) { + test_convert_to_varchar("LARGEINT", 16, 1701411834604690, "1701411834604690", OLAP_SUCCESS); +} + +TEST_F(TestColumn, ConvertFloatToVarchar) { + test_convert_to_varchar("FLOAT", 4, 3.40282e+38, "3.40282e+38", OLAP_SUCCESS); +} + +TEST_F(TestColumn, ConvertDoubleToVarchar) { + test_convert_to_varchar("DOUBLE", 8, 123.456, "123.456", OLAP_SUCCESS); +} + +TEST_F(TestColumn, ConvertDecimalToVarchar) { + decimal12_t val(456, 789000000); + test_convert_to_varchar("Decimal", 12, val, "456.789000000", OLAP_SUCCESS); +} } int main(int argc, char** argv) { diff --git a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md index 3d2befd13639b6..b8974f62218cbb 100644 --- a/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md +++ b/docs/documentation/cn/sql-reference/sql-statements/Data Definition/ALTER TABLE.md @@ -144,6 +144,7 @@ under the License. 4) 分区列不能做任何修改 5) 目前支持以下类型的转换（精度损失由用户保证） TINYINT/SMALLINT/INT/BIGINT 转换成 TINYINT/SMALLINT/INT/BIGINT/DOUBLE。 + TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE/DECIMAL 转换成 VARCHAR LARGEINT 转换成 DOUBLE VARCHAR 支持修改最大长度 VARCHAR 转换成 TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE diff --git a/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md b/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md index 2142a6c5c0a088..bfbed307412805 100644 --- a/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md +++ b/docs/documentation/en/sql-reference/sql-statements/Data Definition/ALTER TABLE_EN.md @@ -145,6 +145,7 @@ under the License. 4) The partition column cannot be modified 5) The following types of conversions are currently supported (accuracy loss is guaranteed by the user) TINYINT/SMALLINT/INT/BIGINT is converted to TINYINT/SMALLINT/INT/BIGINT/DOUBLE. + TINTINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE/DECIMAL is converted to VARCHAR Convert LARGEINT to DOUBLE VARCHAR supports modification of maximum length Convert VARCHAR to TINYINT/SMALLINT/INT/BIGINT/LARGEINT/FLOAT/DOUBLE. diff --git a/fe/src/main/java/org/apache/doris/catalog/ColumnType.java b/fe/src/main/java/org/apache/doris/catalog/ColumnType.java index 78662136178447..557a6f56a14f20 100644 --- a/fe/src/main/java/org/apache/doris/catalog/ColumnType.java +++ b/fe/src/main/java/org/apache/doris/catalog/ColumnType.java @@ -47,19 +47,30 @@ public abstract class ColumnType { schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.BIGINT.ordinal()] = true; schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true; schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.TINYINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true; schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.INT.ordinal()] = true; schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.BIGINT.ordinal()] = true; schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true; schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.SMALLINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true; schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.BIGINT.ordinal()] = true; schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true; schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true; schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.DATE.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.INT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true; schemaChangeMatrix[PrimitiveType.BIGINT.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true; schemaChangeMatrix[PrimitiveType.BIGINT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.BIGINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true; + + schemaChangeMatrix[PrimitiveType.LARGEINT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true; + + schemaChangeMatrix[PrimitiveType.FLOAT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.FLOAT.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true; + + schemaChangeMatrix[PrimitiveType.DOUBLE.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true; schemaChangeMatrix[PrimitiveType.CHAR.ordinal()][PrimitiveType.VARCHAR.ordinal()] = true; schemaChangeMatrix[PrimitiveType.CHAR.ordinal()][PrimitiveType.CHAR.ordinal()] = true; @@ -74,8 +85,8 @@ public abstract class ColumnType { schemaChangeMatrix[PrimitiveType.VARCHAR.ordinal()][PrimitiveType.LARGEINT.ordinal()] = true; schemaChangeMatrix[PrimitiveType.DATETIME.ordinal()][PrimitiveType.DATE.ordinal()] = true; + schemaChangeMatrix[PrimitiveType.DATE.ordinal()][PrimitiveType.DATETIME.ordinal()] = true; - schemaChangeMatrix[PrimitiveType.FLOAT.ordinal()][PrimitiveType.DOUBLE.ordinal()] = true; } static boolean isSchemaChangeAllowed(Type lhs, Type rhs) {