Skip to content

Commit

Permalink
fix: format throw data truncated error (pingcap#5272)
Browse files Browse the repository at this point in the history 
close pingcap#4891
  • Loading branch information
@xzhangxian1008 @Lloyd-Pottiger
xzhangxian1008 authored and Lloyd-Pottiger committed Jul 19, 2022
1 parent d6c36ea commit 0b6f372
Show file tree
Hide file tree
Showing 3 changed files with 61 additions and 12 deletions.
5 changes: 4 additions & 1 deletion dbms/src/Functions/FunctionsString.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4573,7 +4573,9 @@ class FormatImpl : public IFunction
using NumberFieldType = typename NumberType::FieldType;
using NumberColVec = std::conditional_t<IsDecimal<NumberFieldType>, ColumnDecimal<NumberFieldType>, ColumnVector<NumberFieldType>>;
const auto * number_raw = block.getByPosition(arguments[0]).column.get();

TiDBDecimalRoundInfo info{number_type, number_type};
info.output_prec = info.output_prec < 65 ? info.output_prec + 1 : 65;

return getPrecisionType(precision_base_type, [&](const auto & precision_type, bool) {
using PrecisionType = std::decay_t<decltype(precision_type)>;
Expand Down Expand Up @@ -4723,10 +4725,11 @@ class FormatImpl : public IFunction
static void format(
T number,
size_t max_num_decimals,
const TiDBDecimalRoundInfo & info,
TiDBDecimalRoundInfo & info,
ColumnString::Chars_t & res_data,
ColumnString::Offsets & res_offsets)
{
info.output_scale = std::min(max_num_decimals, static_cast<size_t>(info.input_scale));
auto round_number = round(number, max_num_decimals, info);
std::string round_number_str = number2Str(round_number, info);
std::string buffer = Format::apply(round_number_str, max_num_decimals);
Expand Down
19 changes: 8 additions & 11 deletions dbms/src/Functions/tests/gtest_strings_format.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ class StringFormat : public DB::tests::FunctionTest
using FieldType = DecimalField<Decimal>;
using NullableDecimal = Nullable<Decimal>;
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"0.0000", "-0.0120", "0.0120", "12,332.1000", "12,332", "12,332", "12,332.300000000000000000000000000000", "-12,332.30000", "-1,000.0", "-333.33", {}}),
createColumn<Nullable<String>>({"0.0000", "-0.0120", "0.0120", "12,332.1000", "12,332", "12,332", "12,332.300000000000000000000000000000", "-12,332.30000", "-1,000.0", "-333.33", {}, "99,999.9999000000", "100,000.000", "100,000"}),
executeFunction(
func_name,
createColumn<NullableDecimal>(
Expand All @@ -49,8 +49,11 @@ class StringFormat : public DB::tests::FunctionTest
FieldType(static_cast<Native>(-123323000), 4),
FieldType(static_cast<Native>(-9999999), 4),
FieldType(static_cast<Native>(-3333330), 4),
FieldType(static_cast<Native>(0), 0)}),
createColumn<Nullable<Int64>>({4, 4, 4, 4, 0, -1, 31, 5, 1, 2, {}})));
FieldType(static_cast<Native>(0), 0),
FieldType(static_cast<Native>(999999999), 4),
FieldType(static_cast<Native>(999999999), 4),
FieldType(static_cast<Native>(999999999), 4)}),
createColumn<Nullable<Int64>>({4, 4, 4, 4, 0, -1, 31, 5, 1, 2, {}, 10, 3, -5})));
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"12,332.100", "-12,332.300", "-1,000.000", "-333.333"}),
executeFunction(
Expand All @@ -62,8 +65,6 @@ class StringFormat : public DB::tests::FunctionTest
FieldType(static_cast<Native>(-9999999), 4),
FieldType(static_cast<Native>(-3333330), 4)}),
createConstColumn<Nullable<Int16>>(4, 3)));
/// known issue https://github.com/pingcap/tiflash/issues/4891
/*
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"-999.9999", "-1,000", "-1,000", "-999.999900000000000000000000000000", "-999.99990", "-1,000.0", "-1,000.00"}),
executeFunction(
Expand All @@ -74,15 +75,14 @@ class StringFormat : public DB::tests::FunctionTest
FieldType(static_cast<Native>(-9999999), 4)),
createColumn<Nullable<Int32>>({4, 0, -1, 31, 5, 1, 2})));
ASSERT_COLUMN_EQ(
createConstColumn<Nullable<String>>(1, "-1,000.000"),
createConstColumn<String>(1, "-1,000.000"),
executeFunction(
func_name,
createConstColumn<NullableDecimal>(
std::make_tuple(precision, 4),
1,
FieldType(static_cast<Native>(-9999999), 4)),
createConstColumn<Nullable<Int8>>(1, 3)));
*/
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"12,332.1000", "12,332", "12,332.300000000000000000000000000000", "-12,332.30000", "-1,000.0", "-333.33", {}}),
executeFunction(
Expand All @@ -108,8 +108,6 @@ class StringFormat : public DB::tests::FunctionTest
FieldType(static_cast<Native>(-9999999), 4),
FieldType(static_cast<Native>(-3333330), 4)}),
createConstColumn<Nullable<UInt16>>(4, 3)));
/// known issue https://github.com/pingcap/tiflash/issues/4891
/*
ASSERT_COLUMN_EQ(
createColumn<Nullable<String>>({"-999.9999", "-1,000", "-999.999900000000000000000000000000", "-999.99990", "-1,000.0", "-1,000.00"}),
executeFunction(
Expand All @@ -120,15 +118,14 @@ class StringFormat : public DB::tests::FunctionTest
FieldType(static_cast<Native>(-9999999), 4)),
createColumn<Nullable<UInt32>>({4, 0, 31, 5, 1, 2})));
ASSERT_COLUMN_EQ(
createConstColumn<Nullable<String>>(1, "-1,000.000"),
createConstColumn<String>(1, "-1,000.000"),
executeFunction(
func_name,
createConstColumn<NullableDecimal>(
std::make_tuple(precision, 4),
1,
FieldType(static_cast<Native>(-9999999), 4)),
createConstColumn<Nullable<UInt8>>(1, 3)));
*/
}

template <typename Integer>
Expand Down
49 changes: 49 additions & 0 deletions tests/fullstack-test/expr/format.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,52 @@ int_val
1,234.000

mysql> drop table if exists test.t

mysql> create table test.t(id int, value decimal(65,4))
mysql> alter table test.t set tiflash replica 1
mysql> insert into test.t values(1,9999999999999999999999999999999999999999999999999999999999999.9999)

func> wait_table test t

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,-3) as result from test.t
result
10,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,0) as result from test.t
result
10,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,3) as result from test.t
result
10,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000,000.000

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,10) as result from test.t
result
9,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999,999.9999000000


mysql> drop table if exists test.t

mysql> create table test.t(id int, value decimal(7,4))
mysql> alter table test.t set tiflash replica 1
mysql> insert into test.t values(1,999.9999)

func> wait_table test t

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,-2) as result from test.t
result
1,000

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,0) as result from test.t
result
1,000

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,2) as result from test.t
result
1,000.00

mysql> set tidb_enforce_mpp=1; set tidb_isolation_read_engines='tiflash'; select format(value,10) as result from test.t
result
999.9999000000

mysql> drop table if exists test.t

0 comments on commit 0b6f372

Please sign in to comment.