Skip to content

Commit 85b269a

Browse files
rvu1024rvu1024
authored
Merge 32f286b into 84a31b7 
2 parents 84a31b7 + 32f286b commit 85b269a

File tree

5 files changed

+267
-6
lines changed

5 files changed

+267
-6
lines changed

‎ydb/library/yql/minikql/protobuf_udf/proto_builder.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ namespace {
6969
break;
7070

7171
case FieldDescriptor::TYPE_FLOAT:
72-
reflection.AddFloat(&target, &field, item.Get<float>());
72+
reflection.AddFloat(&target, &field, info.YtMode ? float(item.Get<double>()) : item.Get<float>());
7373
break;
7474

7575
case FieldDescriptor::TYPE_INT64:
@@ -140,7 +140,7 @@ namespace {
140140
break;
141141

142142
case FieldDescriptor::TYPE_FLOAT:
143-
reflection.SetFloat(&target, &field, source.Get<float>());
143+
reflection.SetFloat(&target, &field, info.YtMode ? float(source.Get<double>()) : source.Get<float>());
144144
break;
145145

146146
case FieldDescriptor::TYPE_INT64:

‎ydb/library/yql/minikql/protobuf_udf/ut/protobuf_ut.proto‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,15 @@ message TIntegral
2929
message TRepeated
3030
{
3131
repeated int32 Int32Field = 1;
32+
repeated float FloatField = 2;
33+
}
34+
35+
message TWithDefaults
36+
{
37+
optional float DefFloat0 = 1 [default = 0];
38+
optional float DefFloat05 = 2 [default = 0.5];
39+
optional float DefFloatInf = 3 [default = inf];
40+
optional float DefFloatNan = 4 [default = nan];
3241
}
3342

3443
message TWithTypeOptions

‎ydb/library/yql/minikql/protobuf_udf/ut/value_builder_ut.cpp‎

Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
#include "proto_builder.h"
2+
#include "value_builder.h"
3+
4+
#include <ydb/library/yql/minikql/protobuf_udf/ut/protobuf_ut.pb.h>
5+
6+
#include <ydb/library/yql/providers/common/codec/yql_codec.h>
7+
#include <ydb/library/yql/providers/common/codec/yql_codec_buf.h>
8+
#include <ydb/library/yql/minikql/mkql_alloc.h>
9+
#include <ydb/library/yql/minikql/mkql_node.h>
10+
#include <ydb/library/yql/minikql/mkql_type_builder.h>
11+
#include <ydb/library/yql/minikql/mkql_program_builder.h>
12+
#include <ydb/library/yql/minikql/mkql_function_registry.h>
13+
#include <ydb/library/yql/minikql/computation/mkql_value_builder.h>
14+
#include <ydb/library/yql/public/udf/udf_types.h>
15+
16+
#include <google/protobuf/text_format.h>
17+
18+
#include <library/cpp/yson/node/node_io.h>
19+
#include <library/cpp/testing/unittest/registar.h>
20+
21+
#include <util/generic/strbuf.h>
22+
#include <util/generic/string.h>
23+
#include <util/stream/str.h>
24+
#include <util/string/strip.h>
25+
#include <util/memory/tempbuf.h>
26+
27+
28+
using namespace NYql;
29+
using namespace NKikimr::NMiniKQL;
30+
31+
namespace {
32+
33+
struct TSetup {
34+
TSetup()
35+
: Alloc(__LOCATION__)
36+
, Env(Alloc)
37+
, FunctionRegistry(CreateFunctionRegistry(IBuiltinFunctionRegistry::TPtr()))
38+
, TypeInfoHelper(new TTypeInfoHelper())
39+
, FunctionTypeInfoBuilder(Env, TypeInfoHelper, "", nullptr, {})
40+
, PgmBuilder(Env, *FunctionRegistry)
41+
, MemInfo("Test")
42+
, HolderFactory(Alloc.Ref(), MemInfo)
43+
, ValueBuilder(HolderFactory, NUdf::EValidatePolicy::Exception)
44+
{
45+
}
46+
47+
TScopedAlloc Alloc;
48+
TTypeEnvironment Env;
49+
IFunctionRegistry::TPtr FunctionRegistry;
50+
NUdf::ITypeInfoHelper::TPtr TypeInfoHelper;
51+
TFunctionTypeInfoBuilder FunctionTypeInfoBuilder;
52+
TProgramBuilder PgmBuilder;
53+
TMemoryUsageInfo MemInfo;
54+
THolderFactory HolderFactory;
55+
TDefaultValueBuilder ValueBuilder;
56+
};
57+
58+
template <typename TProto>
59+
TString YsonToProtoText(TSetup& setup, NUdf::TProtoInfo& info, TStringBuf yson) {
60+
TStringStream err;
61+
auto val = NCommon::ParseYsonValue(
62+
setup.HolderFactory,
63+
NYT::NodeToYsonString(NYT::NodeFromYsonString(yson), ::NYson::EYsonFormat::Binary),
64+
static_cast<NKikimr::NMiniKQL::TStructType*>(info.StructType),
65+
&err, true);
66+
if (!val) {
67+
throw yexception() << err.Str();
68+
}
69+
TProto proto;
70+
FillProtoFromValue(*val, proto, info);
71+
72+
TString res;
73+
NProtoBuf::TextFormat::Printer printer;
74+
printer.SetSingleLineMode(true);
75+
printer.SetUseShortRepeatedPrimitives(true);
76+
if (!printer.PrintToString(proto, &res)) {
77+
throw yexception() << "Proto printer failed";
78+
}
79+
return StripString(res);
80+
}
81+
82+
class TTestWriter : public NCommon::IBlockWriter {
83+
public:
84+
TTestWriter()
85+
: Buffer_(1024)
86+
{
87+
}
88+
89+
void SetRecordBoundaryCallback(std::function<void()> /*callback*/) override {
90+
}
91+
92+
std::pair<char*, char*> NextEmptyBlock() override {
93+
auto ptr = Buffer_.Current();
94+
return {ptr, ptr + Buffer_.Left()};
95+
}
96+
97+
void ReturnBlock(size_t avail, std::optional<size_t> /*lastRecordBoundary*/) override {
98+
Buffer_.Proceed(avail);
99+
}
100+
101+
void Finish() override {
102+
}
103+
104+
TStringBuf Str() const {
105+
return {Buffer_.Data(), Buffer_.Filled()};
106+
}
107+
108+
private:
109+
TTempBuf Buffer_;
110+
};
111+
112+
template <typename TProto>
113+
TString ProtoTextToYson(TSetup& setup, NUdf::TProtoInfo& info, TStringBuf protoText) {
114+
TProto proto;
115+
if (!NProtoBuf::TextFormat::ParseFromString(TString{protoText}, &proto)) {
116+
throw yexception() << "Failed to parse proto";
117+
}
118+
119+
auto value = FillValueFromProto(proto, &setup.ValueBuilder, info);
120+
TTestWriter out;
121+
NCommon::TOutputBuf buf(out, nullptr);
122+
NCommon::WriteYsonValueInTableFormat(buf, static_cast<NKikimr::NMiniKQL::TStructType*>(info.StructType), value, true);
123+
buf.Finish();
124+
125+
return NYT::NodeToYsonString(NYT::NodeFromYsonString(out.Str()), ::NYson::EYsonFormat::Text);
126+
}
127+
128+
} // unnamed
129+
130+
Y_UNIT_TEST_SUITE(TProtobufValueBuilderTests) {
131+
132+
template <bool YTMODE>
133+
void TestFloatSingle() {
134+
TSetup setup;
135+
NUdf::TProtoInfo info;
136+
info.YtMode = YTMODE;
137+
NUdf::ProtoTypeBuild<NYql::NProtoTest::TIntegral>(setup.FunctionTypeInfoBuilder, &info);
138+
139+
UNIT_ASSERT_VALUES_EQUAL(
140+
YsonToProtoText<NYql::NProtoTest::TIntegral>(setup, info, "{FloatField=0.5}"),
141+
"FloatField: 0.5"
142+
);
143+
UNIT_ASSERT_VALUES_EQUAL(
144+
YsonToProtoText<NYql::NProtoTest::TIntegral>(setup, info, "{FloatField=0.}"),
145+
"FloatField: 0"
146+
);
147+
UNIT_ASSERT_VALUES_EQUAL(
148+
YsonToProtoText<NYql::NProtoTest::TIntegral>(setup, info, "{FloatField=-0.33333333}"),
149+
"FloatField: -0.333333343"
150+
);
151+
UNIT_ASSERT_VALUES_EQUAL(
152+
YsonToProtoText<NYql::NProtoTest::TIntegral>(setup, info, "{FloatField=%-inf}"),
153+
"FloatField: -inf"
154+
);
155+
UNIT_ASSERT_VALUES_EQUAL(
156+
YsonToProtoText<NYql::NProtoTest::TIntegral>(setup, info, "{FloatField=%nan}"),
157+
"FloatField: nan"
158+
);
159+
160+
UNIT_ASSERT_VALUES_EQUAL(
161+
ProtoTextToYson<NYql::NProtoTest::TIntegral>(setup, info, "FloatField: 0.5"),
162+
"[#;#;#;#;#;[0.5];#;#;#;#;#;#;#;#]"
163+
);
164+
UNIT_ASSERT_VALUES_EQUAL(
165+
ProtoTextToYson<NYql::NProtoTest::TIntegral>(setup, info, "FloatField: 0"),
166+
"[#;#;#;#;#;[0.];#;#;#;#;#;#;#;#]"
167+
);
168+
UNIT_ASSERT_VALUES_EQUAL(
169+
ProtoTextToYson<NYql::NProtoTest::TIntegral>(setup, info, "FloatField: -0.33333333"),
170+
"[#;#;#;#;#;[-0.3333333433];#;#;#;#;#;#;#;#]"
171+
);
172+
UNIT_ASSERT_VALUES_EQUAL(
173+
ProtoTextToYson<NYql::NProtoTest::TIntegral>(setup, info, "FloatField: -inf"),
174+
"[#;#;#;#;#;[%-inf];#;#;#;#;#;#;#;#]"
175+
);
176+
UNIT_ASSERT_VALUES_EQUAL(
177+
ProtoTextToYson<NYql::NProtoTest::TIntegral>(setup, info, "FloatField: nan"),
178+
"[#;#;#;#;#;[%nan];#;#;#;#;#;#;#;#]"
179+
);
180+
}
181+
182+
Y_UNIT_TEST(FloatSingleYtMode) {
183+
TestFloatSingle<true>();
184+
}
185+
186+
Y_UNIT_TEST(FloatSingle) {
187+
TestFloatSingle<false>();
188+
}
189+
190+
template <bool YTMODE>
191+
void TestFloatRepeated() {
192+
TSetup setup;
193+
NUdf::TProtoInfo info;
194+
info.YtMode = YTMODE;
195+
info.OptionalLists = true;
196+
NUdf::ProtoTypeBuild<NYql::NProtoTest::TRepeated>(setup.FunctionTypeInfoBuilder, &info);
197+
198+
UNIT_ASSERT_VALUES_EQUAL(
199+
YsonToProtoText<NYql::NProtoTest::TRepeated>(setup, info, "{FloatField=[0.; -0.33333333; 0.5; 1.0; %inf; %nan]}"),
200+
"FloatField: [0, -0.333333343, 0.5, 1, inf, nan]"
201+
);
202+
203+
UNIT_ASSERT_VALUES_EQUAL(
204+
ProtoTextToYson<NYql::NProtoTest::TRepeated>(setup, info, "FloatField: [0, -0.33333333, 0.5, 1, inf, nan]"),
205+
"[[[0.;-0.3333333433;0.5;1.;%inf;%nan]];#]"
206+
);
207+
}
208+
209+
Y_UNIT_TEST(FloatRepeatedYtMode) {
210+
TestFloatRepeated<true>();
211+
}
212+
213+
Y_UNIT_TEST(FloatRepeated) {
214+
TestFloatRepeated<false>();
215+
}
216+
217+
template <bool YTMODE>
218+
void TestFloatDefaults() {
219+
TSetup setup;
220+
NUdf::TProtoInfo info;
221+
info.YtMode = YTMODE;
222+
info.OptionalLists = true;
223+
NUdf::ProtoTypeBuild<NYql::NProtoTest::TWithDefaults>(setup.FunctionTypeInfoBuilder, &info);
224+
225+
UNIT_ASSERT_VALUES_EQUAL(
226+
ProtoTextToYson<NYql::NProtoTest::TWithDefaults>(setup, info, " "),
227+
"[[0.];[0.5];[%inf];[%nan]]"
228+
);
229+
}
230+
231+
Y_UNIT_TEST(FloatDefaultsYtMode) {
232+
TestFloatDefaults<true>();
233+
}
234+
235+
Y_UNIT_TEST(FloatDefaults) {
236+
TestFloatDefaults<false>();
237+
}
238+
};

‎ydb/library/yql/minikql/protobuf_udf/ut/ya.make‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,20 @@ UNITTEST_FOR(ydb/library/yql/minikql/protobuf_udf)
22

33
SRCS(
44
type_builder_ut.cpp
5+
value_builder_ut.cpp
56
protobuf_ut.proto
67
)
78

89
PEERDIR(
910
ydb/library/yql/providers/yt/lib/schema
1011
ydb/library/yql/providers/yt/common
11-
ydb/library/yql/parser/pg_wrapper
1212
ydb/library/yql/public/udf/service/exception_policy
1313
ydb/library/yql/minikql
1414
ydb/library/yql/public/udf
1515
ydb/library/yql/providers/common/schema/mkql
16+
ydb/library/yql/providers/common/codec
17+
ydb/library/yql/sql
18+
ydb/library/yql/sql/pg_dummy
1619
contrib/libs/protobuf
1720

1821
#alice/wonderlogs/protos

‎ydb/library/yql/minikql/protobuf_udf/value_builder.cpp‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -108,9 +108,12 @@ case FieldDescriptor::EProtoCppType: { \
108108
FIELD_TO_VALUE(CPPTYPE_UINT32, GetUInt32);
109109
FIELD_TO_VALUE(CPPTYPE_UINT64, GetUInt64);
110110
FIELD_TO_VALUE(CPPTYPE_DOUBLE, GetDouble);
111-
FIELD_TO_VALUE(CPPTYPE_FLOAT, GetFloat);
112111
FIELD_TO_VALUE(CPPTYPE_BOOL, GetBool);
113112

113+
case FieldDescriptor::CPPTYPE_FLOAT: {
114+
const auto f = r->GetFloat(proto, fd);
115+
return info.YtMode ? TUnboxedValuePod(double(f)) : TUnboxedValuePod(f);
116+
}
114117
case FieldDescriptor::CPPTYPE_ENUM: {
115118
return CreateEnumValue(valueBuilder, r->GetEnum(proto, fd), info.EnumFormat, fieldFlags);
116119
}
@@ -150,9 +153,12 @@ case FieldDescriptor::EProtoCppType: { \
150153
DEFAULT_TO_VALUE(CPPTYPE_UINT32, default_value_uint32);
151154
DEFAULT_TO_VALUE(CPPTYPE_UINT64, default_value_uint64);
152155
DEFAULT_TO_VALUE(CPPTYPE_DOUBLE, default_value_double);
153-
DEFAULT_TO_VALUE(CPPTYPE_FLOAT, default_value_float);
154156
DEFAULT_TO_VALUE(CPPTYPE_BOOL, default_value_bool);
155157

158+
case FieldDescriptor::CPPTYPE_FLOAT: {
159+
const auto f = fd->default_value_float();
160+
return info.YtMode ? TUnboxedValuePod(double(f)) : TUnboxedValuePod(f);
161+
}
156162
case FieldDescriptor::CPPTYPE_ENUM:
157163
return CreateEnumValue(valueBuilder, fd->default_value_enum(), info.EnumFormat, fieldFlags);
158164

@@ -190,9 +196,14 @@ case FieldDescriptor::EProtoCppType: { \
190196
REPEATED_FIELD_TO_VALUE(CPPTYPE_UINT32, GetRepeatedUInt32);
191197
REPEATED_FIELD_TO_VALUE(CPPTYPE_UINT64, GetRepeatedUInt64);
192198
REPEATED_FIELD_TO_VALUE(CPPTYPE_DOUBLE, GetRepeatedDouble);
193-
REPEATED_FIELD_TO_VALUE(CPPTYPE_FLOAT, GetRepeatedFloat);
194199
REPEATED_FIELD_TO_VALUE(CPPTYPE_BOOL, GetRepeatedBool);
195200

201+
case FieldDescriptor::CPPTYPE_FLOAT:
202+
for (int i = 0; i < endI; ++i) {
203+
const auto f = r->GetRepeatedFloat(proto, fd, i);
204+
*items++ = info.YtMode ? TUnboxedValuePod(double(f)) : TUnboxedValuePod(f);
205+
}
206+
break;
196207
case FieldDescriptor::CPPTYPE_ENUM:
197208
for (int i = 0; i < endI; ++i) {
198209
*items++ = CreateEnumValue(valueBuilder, r->GetRepeatedEnum(proto, fd, i), info.EnumFormat, fieldFlags);

0 commit comments

Comments
 (0)