Add decimal128 type support with big precision (#1607)

Author: kinash-varvara

ydb/library/yql/parser/pg_wrapper/arrow.cpp

@@ -1,5 +1,6 @@
 #include "arrow.h"
 #include "arrow_impl.h"
+#include "ydb/library/yql/minikql/defs.h"
 #include <ydb/library/yql/parser/pg_wrapper/interface/arrow.h>
 #include <ydb/library/yql/parser/pg_wrapper/interface/utils.h>
 #include <ydb/library/yql/minikql/mkql_node_cast.h>
@@ -183,14 +184,17 @@ std::shared_ptr<arrow::Array> PgDecimal128ConvertNumeric(const std::shared_ptr<a
     size_t length = data->length;
     arrow::BinaryBuilder builder;
 
+    bool error;
+    Numeric high_bits_mul = numeric_mul_opt_error(int64_to_numeric(int64_t(1) << 62), int64_to_numeric(4), &error);
+
     auto input = data->GetValues<arrow::Decimal128>(1);
     for (size_t i = 0; i < length; ++i) {
         if (value->IsNull(i)) {
             ARROW_OK(builder.AppendNull());
             continue;
         }
 
-        Numeric v = PgDecimal128ToNumeric(input[i].high_bits(), input[i].low_bits(), precision, scale);
+        Numeric v = PgDecimal128ToNumeric(input[i], precision, scale, high_bits_mul);
 
         auto datum = NumericGetDatum(v);
         auto ptr = (char*)datum;
@@ -204,8 +208,22 @@ std::shared_ptr<arrow::Array> PgDecimal128ConvertNumeric(const std::shared_ptr<a
     return ret;
 }
 
-Numeric PgDecimal128ToNumeric(int64_t high_bits, uint64_t low_bits, int32_t precision, int32_t scale) {
-    Numeric res = int64_div_fast_to_numeric(low_bits, scale);
+Numeric PgDecimal128ToNumeric(arrow::Decimal128 value, int32_t precision, int32_t scale, Numeric high_bits_mul) {
+    uint64_t low_bits = value.low_bits();
+    int64 high_bits = value.high_bits();
+
+    if (low_bits > INT64_MAX){
+        high_bits += 1;
+    }
+
+    bool error;
+    Numeric low_bits_res  = int64_div_fast_to_numeric(low_bits, scale);
+    Numeric high_bits_res = numeric_mul_opt_error(int64_div_fast_to_numeric(high_bits, scale), high_bits_mul, &error);
+    MKQL_ENSURE(error == false, "Bad numeric multiplication.");
+    
+    Numeric res = numeric_add_opt_error(high_bits_res,  low_bits_res, &error);
+    MKQL_ENSURE(error == false, "Bad numeric addition.");
+
     return res;
 }
 

ydb/library/yql/parser/pg_wrapper/arrow_impl.h

@@ -11,7 +11,7 @@ extern "C" {
 namespace NYql {
 
 Numeric PgFloatToNumeric(double item, ui64 scale, int digits);
-Numeric PgDecimal128ToNumeric(int64_t high_bits, uint64_t low_bits, int32_t precision, int32_t scale);
+Numeric PgDecimal128ToNumeric(arrow::Decimal128 val, int32_t precision, int32_t scale, Numeric high_bits_mul);
 TColumnConverter BuildPgColumnConverter(const std::shared_ptr<arrow::DataType>& originalType, NKikimr::NMiniKQL::TPgType* targetType);
 
 template<typename T>

ydb/library/yql/parser/pg_wrapper/ut/arrow_ut.cpp

@@ -185,6 +185,56 @@ Y_UNIT_TEST(PgConvertNumericDecimal128Scale5) {
     checkResult<false>(expected, result, &reader, numeric_out);
 }
 
+Y_UNIT_TEST(PgConvertNumericDecimal128BigScale3) {
+    TArenaMemoryContext arena;
+
+    int32_t precision = 20;
+    int32_t scale = 3;
+    std::shared_ptr<arrow::DataType> type(new arrow::Decimal128Type(precision, scale));
+    arrow::Decimal128Builder builder(type);
+
+    const char* expected[] = {
+        "36893488147419103.245", "-36893488147419103.245", nullptr
+    };
+
+    ARROW_OK(builder.Append(arrow::Decimal128::FromString("36893488147419103.245").ValueOrDie()));
+    ARROW_OK(builder.Append(arrow::Decimal128::FromString("-36893488147419103.245").ValueOrDie()));
+    ARROW_OK(builder.AppendNull());
+
+    std::shared_ptr<arrow::Array> array;
+    ARROW_OK(builder.Finish(&array));
+
+    auto result = PgDecimal128ConvertNumeric(array, precision, scale);
+
+    NYql::NUdf::TStringBlockReader<arrow::BinaryType, true> reader;
+    checkResult<false>(expected, result, &reader, numeric_out);
+}
+
+Y_UNIT_TEST(PgConvertNumericDecimal128BigScale1) {
+    TArenaMemoryContext arena;
+
+    int32_t precision = 26;
+    int32_t scale = 1;
+    std::shared_ptr<arrow::DataType> type(new arrow::Decimal128Type(precision, scale));
+    arrow::Decimal128Builder builder(type);
+
+    const char* expected[] = {
+        "3868562622766813359059763.2", "-3868562622766813359059763.2", nullptr
+    };
+
+    ARROW_OK(builder.Append(arrow::Decimal128::FromString("3868562622766813359059763.2").ValueOrDie()));
+    ARROW_OK(builder.Append(arrow::Decimal128::FromString("-3868562622766813359059763.2").ValueOrDie()));
+    ARROW_OK(builder.AppendNull());
+
+    std::shared_ptr<arrow::Array> array;
+    ARROW_OK(builder.Finish(&array));
+
+    auto result = PgDecimal128ConvertNumeric(array, precision, scale);
+
+    NYql::NUdf::TStringBlockReader<arrow::BinaryType, true> reader;
+    checkResult<false>(expected, result, &reader, numeric_out);
+}
+
 Y_UNIT_TEST(PgConvertNumericInt) {
     TArenaMemoryContext arena;